Skip to content
Snippets Groups Projects
Commit 349387c4 authored by Madelein van Niekerk's avatar Madelein van Niekerk :one:
Browse files

Merge branch '250699-epic-index' into 'master'

Draft: Create Epic Elasticsearch index

See merge request gitlab-org/gitlab!121635



Merged-by: default avatarMadelein van Niekerk <mvanniekerk@gitlab.com>
parents f9822a1f ecaf5903
No related branches found
No related tags found
No related merge requests found
Showing
with 438 additions and 0 deletions
......@@ -54,6 +54,7 @@ Search/NamespacedClass:
- 'ee/app/workers/concerns/elastic/bulk_cron_worker.rb'
- 'ee/app/workers/concerns/elastic/indexing_control.rb'
- 'ee/app/workers/concerns/elastic/migration_backfill_helper.rb'
- 'ee/app/workers/concerns/elastic/migration_create_index.rb'
- 'ee/app/workers/concerns/elastic/migration_helper.rb'
- 'ee/app/workers/concerns/elastic/migration_obsolete.rb'
- 'ee/app/workers/concerns/elastic/migration_options.rb'
......@@ -92,6 +93,9 @@ Search/NamespacedClass:
- 'ee/lib/elastic/latest/config.rb'
- 'ee/lib/elastic/latest/custom_language_analyzers.rb'
- 'ee/lib/elastic/latest/document_should_be_deleted_from_index_error.rb'
- 'ee/lib/elastic/latest/epic_class_proxy.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/epic_instance_proxy.rb'
- 'ee/lib/elastic/latest/git_class_proxy.rb'
- 'ee/lib/elastic/latest/git_instance_proxy.rb'
- 'ee/lib/elastic/latest/issue_class_proxy.rb'
......@@ -129,6 +133,8 @@ Search/NamespacedClass:
- 'ee/lib/elastic/v12p1/application_class_proxy.rb'
- 'ee/lib/elastic/v12p1/application_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/config.rb'
- 'ee/lib/elastic/v12p1/epic_class_proxy.rb'
- 'ee/lib/elastic/v12p1/epic_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_class_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/merge_request_class_proxy.rb'
......
......@@ -233,6 +233,7 @@ Style/RedundantSelf:
- 'ee/lib/ee/model.rb'
- 'ee/lib/elastic/instance_proxy_util.rb'
- 'ee/lib/elastic/latest/commit_config.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/issue_config.rb'
- 'ee/lib/elastic/latest/merge_request_config.rb'
- 'ee/lib/elastic/latest/note_config.rb'
......
---
name: search_index_curation_epics
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/413605
milestone: '16.1'
type: ops
group: group::global search
default_enabled: false
......@@ -23,6 +23,7 @@ module Epic
include EachBatch
include ::Exportable
include Epics::MetadataCacheUpdate
include Elastic::ApplicationVersionedSearch
DEFAULT_COLOR = ::Gitlab::Color.of('#1068bf')
MAX_HIERARCHY_DEPTH = 7
......@@ -325,6 +326,11 @@ def search(query)
fuzzy_search(query, [:title, :description])
end
override :use_separate_indices?
def use_separate_indices?
true
end
def ids_for_base_and_decendants(epic_ids)
::Gitlab::ObjectHierarchy.new(self.id_in(epic_ids)).base_and_descendants.pluck(:id)
end
......@@ -519,6 +525,10 @@ def validate_parent
validate_parent_epic
end
def use_elasticsearch?
group&.use_elasticsearch?
end
def issues_readable_by(current_user, preload: nil)
related_issues = self.class.related_issues(ids: id, preload: preload)
......
......@@ -2,6 +2,7 @@
module Elastic
class ProcessInitialBookkeepingService < Elastic::ProcessBookkeepingService
INDEXED_GROUP_ASSOCIATIONS = [:epics].freeze
INDEXED_PROJECT_ASSOCIATIONS = [
:issues,
:merge_requests,
......@@ -36,6 +37,14 @@ def backfill_projects!(*projects)
end
end
end
def backfill_group_associations!(*groups)
groups.each do |group|
raise ArgumentError, 'This method only accepts Groups' unless group.is_a?(Group)
maintain_indexed_associations(group, INDEXED_GROUP_ASSOCIATIONS)
end
end
end
end
end
# frozen_string_literal: true
module Elastic
module MigrationCreateIndex
include Elastic::MigrationHelper
def migrate
reindexing_cleanup!
log "Creating standalone #{document_type} index #{new_index_name}"
helper.create_standalone_indices(target_classes: [target_class])
rescue StandardError => e
log('Failed to create index', error: e.message)
raise StandardError, e.message
end
def completed?
helper.index_exists?(index_name: new_index_name)
end
def target_class
raise NotImplementedError
end
def document_type
raise NotImplementedError
end
end
end
......@@ -12,6 +12,7 @@ def perform(id)
namespace = Namespace.find(id)
update_users_through_membership(namespace)
update_epics(namespace)
end
def update_users_through_membership(namespace)
......@@ -31,6 +32,12 @@ def update_users_through_membership(namespace)
# rubocop:enable CodeReuse/ActiveRecord
end
def update_epics(namespace)
return unless namespace.type == 'Group'
Elastic::ProcessBookkeepingService.maintain_indexed_associations(namespace, [:epics])
end
def group_and_descendants_user_ids(namespace)
namespace.self_and_descendants.flat_map(&:user_ids)
end
......
# frozen_string_literal: true
class CreateEpicIndex < Elastic::Migration
include Elastic::MigrationCreateIndex
retry_on_failure
def document_type
:epic
end
def target_class
Epic
end
end
# frozen_string_literal: true
module Elastic
module Latest
class EpicClassProxy < ApplicationClassProxy
def elastic_search(query, options: {})
raise NotImplementedError
end
def preload_indexing_data(relation)
# rubocop: disable CodeReuse/ActiveRecord
relation.includes(
:author,
:labels,
:group,
:start_date_sourcing_epic,
:due_date_sourcing_epic,
:start_date_sourcing_milestone,
:due_date_sourcing_milestone
)
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
module EpicConfig
extend Elasticsearch::Model::Indexing::ClassMethods
extend Elasticsearch::Model::Naming::ClassMethods
self.index_name = [Rails.application.class.module_parent_name.downcase, Rails.env, 'epics'].join('-')
settings Elastic::Latest::Config.settings.to_hash.deep_merge(
index: {
number_of_shards: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_shards },
number_of_replicas: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_replicas }
}
)
mappings dynamic: 'strict' do
indexes :type, type: :keyword
indexes :id, type: :integer
indexes :iid, type: :integer
indexes :group_id, type: :integer
indexes :traversal_ids, type: :keyword
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :start_date, type: :date
indexes :due_date, type: :date
indexes :title, type: :text, index_options: 'positions'
indexes :description, type: :text, index_options: 'positions'
indexes :state, type: :keyword
indexes :confidential, type: :boolean
indexes :author_id, type: :integer
indexes :label_ids, type: :keyword
indexes :schema_version, type: :short
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
class EpicInstanceProxy < ApplicationInstanceProxy
def as_indexed_json(_options = {})
data = {}
[
:id,
:iid,
:created_at,
:updated_at,
:group_id,
:title,
:description,
:state,
:confidential,
:author_id
].each do |attr|
data[attr.to_s] = safely_read_attribute_for_elasticsearch(attr)
end
data['start_date'] = target.start_date || target.start_date_from_inherited_source
data['due_date'] = target.end_date || target.due_date_from_inherited_source
data['traversal_ids'] = target.group.elastic_namespace_ancestry
data['label_ids'] = target.label_ids.map(&:to_s)
# Schema version. The format is Date.today.strftime('%y_%m')
# Please update if you're changing the schema of the document
data['schema_version'] = 23_05
data.merge(generic_attributes)
end
def generic_attributes
{ 'type' => es_type }
end
def es_parent
nil
end
end
end
end
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicClassProxy = Elastic::Latest::EpicClassProxy
end
end
# rubocop:enable Naming/FileName
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicInstanceProxy = Elastic::Latest::EpicInstanceProxy
end
end
# rubocop:enable Naming/FileName
......@@ -11,6 +11,7 @@ class Helper
Milestone,
ProjectWiki,
Repository,
Epic,
User
].freeze
......@@ -19,6 +20,7 @@ class Helper
Note,
MergeRequest,
Commit,
Epic,
User,
Wiki
].freeze
......
......@@ -25,6 +25,7 @@ namespace :gitlab do
Rake::Task["gitlab:elastic:index_projects"].invoke
Rake::Task["gitlab:elastic:index_snippets"].invoke
Rake::Task["gitlab:elastic:index_users"].invoke
Rake::Task["gitlab:elastic:index_epics"].invoke
end
desc 'GitLab | Elasticsearch | Enable Elasticsearch search'
......@@ -101,6 +102,24 @@ namespace :gitlab do
logger.info("Indexing users... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Index epics"
task index_epics: :environment do
logger = Logger.new($stdout)
logger.info("Indexing epics...")
groups = Group.all
if ::Gitlab::CurrentSettings.elasticsearch_limit_indexing?
groups.merge!(::Gitlab::CurrentSettings.elasticsearch_limited_namespaces)
end
groups.each_batch do |batch|
::Elastic::ProcessInitialBookkeepingService.backfill_group_associations!(*batch)
end
logger.info("Indexing epics... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Create empty indexes and assigns an alias for each"
task create_empty_index: [:environment] do |t, args|
with_alias = ENV["SKIP_ALIAS"].nil?
......
# frozen_string_literal: true
require 'spec_helper'
require_relative 'migration_shared_examples'
require File.expand_path('ee/elastic/migrate/20230518135700_create_epic_index.rb')
RSpec.describe CreateEpicIndex, feature_category: :global_search do
it_behaves_like 'creates a new index', 20221018125700, Epic
end
......@@ -200,6 +200,66 @@ def update_by_query(objects, script)
end
end
RSpec.shared_examples 'creates a new index' do |version, klass|
let(:helper) { Gitlab::Elastic::Helper.new }
before do
allow(subject).to receive(:helper).and_return(helper)
end
subject { described_class.new(version) }
describe '#migrate' do
it 'logs a message and creates a standalone index' do
expect(subject).to receive(:log).with(/Creating standalone .* index/)
expect(helper).to receive(:create_standalone_indices).with(target_classes: [klass]).and_return(true).once
subject.migrate
end
describe 'reindexing_cleanup!' do
context 'when the index already exists' do
before do
allow(helper).to receive(:index_exists?).and_return(true)
allow(helper).to receive(:create_standalone_indices).and_return(true)
end
it 'deletes the index' do
expect(helper).to receive(:delete_index).once
subject.migrate
end
end
end
context 'when an error is raised' do
let(:error) { 'oops' }
before do
allow(helper).to receive(:create_standalone_indices).and_raise(StandardError, error)
allow(subject).to receive(:log).and_return(true)
end
it 'logs a message and raises an error' do
expect(subject).to receive(:log).with(/Failed to create index/, error: error)
expect { subject.migrate }.to raise_error(StandardError, error)
end
end
end
describe '#completed?' do
[true, false].each do |matcher|
it 'returns true if the index exists' do
allow(helper).to receive(:create_standalone_indices).and_return(true)
allow(helper).to receive(:index_exists?).with(index_name: /gitlab-test-/).and_return(matcher)
expect(subject.completed?).to eq(matcher)
end
end
end
end
RSpec.shared_examples 'a deprecated Advanced Search migration' do |version|
subject { described_class.new(version) }
......
# frozen_string_literal: true
require 'spec_helper'
require_relative './config_shared_examples'
RSpec.describe Elastic::Latest::EpicConfig, feature_category: :global_search do
describe '.settings' do
it_behaves_like 'config settings return correct values'
end
describe '.mappings' do
it 'returns config' do
expect(described_class.mapping).to be_a(Elasticsearch::Model::Indexing::Mappings)
end
end
describe '.index_name' do
it 'includes' do
expect(described_class.index_name).to include('-epics')
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Elastic::Latest::EpicInstanceProxy, feature_category: :global_search do
let_it_be(:group) { create(:group) }
let_it_be(:label) { create(:group_label, group: group) }
let_it_be(:epic) { create(:labeled_epic, :use_fixed_dates, :opened, group: group, labels: [label]) }
subject { described_class.new(epic) }
describe '#as_indexed_json' do
let(:result) { subject.as_indexed_json.with_indifferent_access }
it 'serializes the object as a hash' do
expect(result).to include(
id: epic.id,
iid: epic.iid,
title: epic.title,
description: epic.description,
author_id: epic.author_id,
created_at: epic.created_at,
updated_at: epic.updated_at,
start_date: epic.start_date,
due_date: epic.due_date,
traversal_ids: group.elastic_namespace_ancestry,
state: 'opened',
label_ids: [label.id.to_s],
schema_version: 2305,
type: 'epic'
)
end
context 'with start date inherited date from child epic and due date inherited from milestone' do
let_it_be(:epic) { create(:epic) }
let_it_be(:child_epic) { create(:epic, :use_fixed_dates) }
let_it_be(:milestone) { create(:milestone, :with_dates) }
before do
epic.start_date_sourcing_epic = child_epic
epic.due_date_sourcing_milestone = milestone
epic.save!
end
it 'sets start and due dates to inherited dates' do
expect(result[:start_date]).to eq(child_epic.start_date)
expect(result[:due_date]).to eq(milestone.due_date)
end
end
end
describe '#es_parent' do
it 'is nil so that elasticsearch routing is disabled' do
expect(subject.es_parent).to be_nil
end
end
end
......@@ -1184,6 +1184,56 @@ def as_item(item)
end
end
describe 'ES related specs' do
let_it_be(:epic) { create(:epic, group: group) }
context 'when the group has use_elasticsearch? as true' do
before do
allow(group).to receive(:use_elasticsearch?).and_return(true)
end
it 'use_elasticsearch? is true' do
expect(epic).to be_use_elasticsearch
end
context 'with elasticsearch enabled' do
before do
allow(Gitlab::CurrentSettings.current_application_settings)
.to receive(:elasticsearch_indexing?).and_return(true)
end
it 'calls ::Elastic::ProcessBookkeepingService.track! when the epic is updated' do
expect(Elastic::ProcessBookkeepingService).to receive(:track!).with(*epic).once
epic.update!(title: 'A new title')
end
end
end
context 'when the group has use_elasticsearch? as false' do
before do
allow(group).to receive(:use_elasticsearch?).and_return(false)
end
it 'use_elasticsearch? is false' do
expect(epic).not_to be_use_elasticsearch
end
context 'with elasticsearch enabled' do
before do
allow(Gitlab::CurrentSettings.current_application_settings)
.to receive(:elasticsearch_indexing?).and_return(true)
end
it 'does not call ::Elastic::ProcessBookkeepingService.track! when the epic is updated' do
expect(Elastic::ProcessBookkeepingService).not_to receive(:track!).with(*epic)
epic.update!(title: 'A new title')
end
end
end
end
describe '#epic_link_type' do
let_it_be(:source_epic) { create(:epic, group: group) }
let_it_be(:target_epic) { create(:epic, group: group) }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment