Skip to content
Snippets Groups Projects
Commit fd28bf38 authored by Madelein van Niekerk's avatar Madelein van Niekerk :one:
Browse files

Merge branch '250699-epic-index' into 'master'

Draft: Create Epic Elasticsearch index

See merge request gitlab-org/gitlab!121635



Merged-by: default avatarMadelein van Niekerk <mvanniekerk@gitlab.com>
parents 823d040d d5a5bcf1
No related branches found
No related tags found
No related merge requests found
Showing
with 355 additions and 6 deletions
......@@ -54,6 +54,7 @@ Search/NamespacedClass:
- 'ee/app/workers/concerns/elastic/bulk_cron_worker.rb'
- 'ee/app/workers/concerns/elastic/indexing_control.rb'
- 'ee/app/workers/concerns/elastic/migration_backfill_helper.rb'
- 'ee/app/workers/concerns/elastic/migration_create_index.rb'
- 'ee/app/workers/concerns/elastic/migration_helper.rb'
- 'ee/app/workers/concerns/elastic/migration_obsolete.rb'
- 'ee/app/workers/concerns/elastic/migration_options.rb'
......@@ -92,6 +93,9 @@ Search/NamespacedClass:
- 'ee/lib/elastic/latest/config.rb'
- 'ee/lib/elastic/latest/custom_language_analyzers.rb'
- 'ee/lib/elastic/latest/document_should_be_deleted_from_index_error.rb'
- 'ee/lib/elastic/latest/epic_class_proxy.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/epic_instance_proxy.rb'
- 'ee/lib/elastic/latest/git_class_proxy.rb'
- 'ee/lib/elastic/latest/git_instance_proxy.rb'
- 'ee/lib/elastic/latest/issue_class_proxy.rb'
......@@ -129,6 +133,8 @@ Search/NamespacedClass:
- 'ee/lib/elastic/v12p1/application_class_proxy.rb'
- 'ee/lib/elastic/v12p1/application_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/config.rb'
- 'ee/lib/elastic/v12p1/epic_class_proxy.rb'
- 'ee/lib/elastic/v12p1/epic_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_class_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/merge_request_class_proxy.rb'
......
......@@ -233,6 +233,7 @@ Style/RedundantSelf:
- 'ee/lib/ee/model.rb'
- 'ee/lib/elastic/instance_proxy_util.rb'
- 'ee/lib/elastic/latest/commit_config.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/issue_config.rb'
- 'ee/lib/elastic/latest/merge_request_config.rb'
- 'ee/lib/elastic/latest/note_config.rb'
......
---
name: search_index_curation_epics
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/413605
milestone: '16.1'
type: ops
group: group::global search
default_enabled: false
......@@ -23,6 +23,7 @@ module Epic
include EachBatch
include ::Exportable
include Epics::MetadataCacheUpdate
include Elastic::ApplicationVersionedSearch
DEFAULT_COLOR = ::Gitlab::Color.of('#1068bf')
MAX_HIERARCHY_DEPTH = 7
......@@ -325,6 +326,11 @@ def search(query)
fuzzy_search(query, [:title, :description])
end
override :use_separate_indices?
def use_separate_indices?
true
end
def ids_for_base_and_decendants(epic_ids)
::Gitlab::ObjectHierarchy.new(self.id_in(epic_ids)).base_and_descendants.pluck(:id)
end
......@@ -519,6 +525,10 @@ def validate_parent
validate_parent_epic
end
def use_elasticsearch?
group&.use_elasticsearch?
end
def issues_readable_by(current_user, preload: nil)
related_issues = self.class.related_issues(ids: id, preload: preload)
......
......@@ -2,6 +2,7 @@
module Elastic
class ProcessInitialBookkeepingService < Elastic::ProcessBookkeepingService
INDEXED_GROUP_ASSOCIATIONS = [:epics].freeze
INDEXED_PROJECT_ASSOCIATIONS = [
:issues,
:merge_requests,
......@@ -36,6 +37,14 @@ def backfill_projects!(*projects)
end
end
end
def backfill_group_associations!(*groups)
groups.each do |group|
raise ArgumentError, 'This method only accepts Groups' unless group.is_a?(Group)
maintain_indexed_associations(group, INDEXED_GROUP_ASSOCIATIONS)
end
end
end
end
end
# frozen_string_literal: true
module Elastic
module MigrationCreateIndex
include Elastic::MigrationHelper
def migrate
reindexing_cleanup!
log "Creating standalone #{document_type} index #{new_index_name}"
helper.create_standalone_indices(target_classes: [target_class])
rescue StandardError => e
log('Failed to create index', error: e.message)
raise StandardError, e.message
end
def completed?
helper.index_exists?(index_name: new_index_name)
end
def target_class
raise NotImplementedError
end
def document_type
raise NotImplementedError
end
end
end
......@@ -12,6 +12,7 @@ def perform(id)
namespace = Namespace.find(id)
update_users_through_membership(namespace)
update_epics(namespace)
end
def update_users_through_membership(namespace)
......@@ -31,6 +32,12 @@ def update_users_through_membership(namespace)
# rubocop:enable CodeReuse/ActiveRecord
end
def update_epics(namespace)
return unless namespace.type == 'Group'
Elastic::ProcessBookkeepingService.maintain_indexed_associations(namespace, [:epics])
end
def group_and_descendants_user_ids(namespace)
namespace.self_and_descendants.flat_map(&:user_ids)
end
......
# frozen_string_literal: true
class CreateEpicIndex < Elastic::Migration
include Elastic::MigrationCreateIndex
retry_on_failure
def document_type
:epic
end
def target_class
Epic
end
end
......@@ -6,7 +6,9 @@ class ApplicationInstanceProxy < Elasticsearch::Model::Proxy::InstanceMethodsPro
include InstanceProxyUtil
def es_parent
"project_#{target.project_id}" unless target.is_a?(Project) || target&.project_id.nil?
return if target.is_a?(Project)
return "project_#{target.project_id}" if target.respond_to?(:project_id) && target.project_id.present?
return "group_#{target.group_id}" if target.respond_to?(:group_id) && target.group_id.present?
end
def es_type
......
# frozen_string_literal: true
module Elastic
module Latest
class EpicClassProxy < ApplicationClassProxy
def elastic_search(query, options: {})
raise NotImplementedError
end
def preload_indexing_data(relation)
# rubocop: disable CodeReuse/ActiveRecord
relation.includes(
:author,
:labels,
:group,
:start_date_sourcing_epic,
:due_date_sourcing_epic,
:start_date_sourcing_milestone,
:due_date_sourcing_milestone
)
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
module EpicConfig
extend Elasticsearch::Model::Indexing::ClassMethods
extend Elasticsearch::Model::Naming::ClassMethods
self.index_name = [Rails.application.class.module_parent_name.downcase, Rails.env, 'epics'].join('-')
settings Elastic::Latest::Config.settings.to_hash.deep_merge(
index: {
number_of_shards: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_shards },
number_of_replicas: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_replicas }
}
)
mappings dynamic: 'strict' do
indexes :id, type: :integer
indexes :iid, type: :integer
indexes :group_id, type: :integer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :title, type: :text, index_options: 'positions'
indexes :description, type: :text, index_options: 'positions'
indexes :state, type: :keyword
indexes :confidential, type: :boolean
indexes :author_id, type: :integer
indexes :label_ids, type: :keyword
indexes :start_date, type: :date
indexes :due_date, type: :date
indexes :traversal_ids, type: :keyword
indexes :hashed_root_namespace_id, type: :integer
indexes :schema_version, type: :short
indexes :type, type: :keyword
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
class EpicInstanceProxy < ApplicationInstanceProxy
def as_indexed_json(_options = {})
data = {}
[
:id,
:iid,
:group_id,
:created_at,
:updated_at,
:title,
:description,
:state,
:confidential,
:author_id
].each do |attr|
data[attr.to_s] = safely_read_attribute_for_elasticsearch(attr)
end
data['label_ids'] = target.label_ids.map(&:to_s)
data['start_date'] = target.start_date || target.start_date_from_inherited_source
data['due_date'] = target.end_date || target.due_date_from_inherited_source
data['traversal_ids'] = target.group.elastic_namespace_ancestry
data['hashed_root_namespace_id'] = target.group.hashed_root_namespace_id
# Schema version. The format is Date.today.strftime('%y_%m')
# Please update if you're changing the schema of the document
data['schema_version'] = 23_05
data.merge(generic_attributes)
end
def generic_attributes
super.except('join_field')
end
end
end
end
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicClassProxy = Elastic::Latest::EpicClassProxy
end
end
# rubocop:enable Naming/FileName
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicInstanceProxy = Elastic::Latest::EpicInstanceProxy
end
end
# rubocop:enable Naming/FileName
......@@ -11,6 +11,7 @@ class Helper
Milestone,
ProjectWiki,
Repository,
Epic,
User
].freeze
......@@ -19,6 +20,7 @@ class Helper
Note,
MergeRequest,
Commit,
Epic,
User,
Wiki
].freeze
......
......@@ -25,6 +25,7 @@ namespace :gitlab do
Rake::Task["gitlab:elastic:index_projects"].invoke
Rake::Task["gitlab:elastic:index_snippets"].invoke
Rake::Task["gitlab:elastic:index_users"].invoke
Rake::Task["gitlab:elastic:index_epics"].invoke
end
desc 'GitLab | Elasticsearch | Enable Elasticsearch search'
......@@ -101,6 +102,24 @@ namespace :gitlab do
logger.info("Indexing users... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Index epics"
task index_epics: :environment do
logger = Logger.new($stdout)
logger.info("Indexing epics...")
groups = Group.all
if ::Gitlab::CurrentSettings.elasticsearch_limit_indexing?
groups.merge!(::Gitlab::CurrentSettings.elasticsearch_limited_namespaces)
end
groups.each_batch do |batch|
::Elastic::ProcessInitialBookkeepingService.backfill_group_associations!(*batch)
end
logger.info("Indexing epics... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Create empty indexes and assigns an alias for each"
task create_empty_index: [:environment] do |t, args|
with_alias = ENV["SKIP_ALIAS"].nil?
......
# frozen_string_literal: true
require 'spec_helper'
require_relative 'migration_shared_examples'
require File.expand_path('ee/elastic/migrate/20230518135700_create_epic_index.rb')
RSpec.describe CreateEpicIndex, feature_category: :global_search do
it_behaves_like 'creates a new index', 20221018125700, Epic
end
......@@ -200,6 +200,66 @@ def update_by_query(objects, script)
end
end
RSpec.shared_examples 'creates a new index' do |version, klass|
let(:helper) { Gitlab::Elastic::Helper.new }
before do
allow(subject).to receive(:helper).and_return(helper)
end
subject { described_class.new(version) }
describe '#migrate' do
it 'logs a message and creates a standalone index' do
expect(subject).to receive(:log).with(/Creating standalone .* index/)
expect(helper).to receive(:create_standalone_indices).with(target_classes: [klass]).and_return(true).once
subject.migrate
end
describe 'reindexing_cleanup!' do
context 'when the index already exists' do
before do
allow(helper).to receive(:index_exists?).and_return(true)
allow(helper).to receive(:create_standalone_indices).and_return(true)
end
it 'deletes the index' do
expect(helper).to receive(:delete_index).once
subject.migrate
end
end
end
context 'when an error is raised' do
let(:error) { 'oops' }
before do
allow(helper).to receive(:create_standalone_indices).and_raise(StandardError, error)
allow(subject).to receive(:log).and_return(true)
end
it 'logs a message and raises an error' do
expect(subject).to receive(:log).with(/Failed to create index/, error: error)
expect { subject.migrate }.to raise_error(StandardError, error)
end
end
end
describe '#completed?' do
[true, false].each do |matcher|
it 'returns true if the index exists' do
allow(helper).to receive(:create_standalone_indices).and_return(true)
allow(helper).to receive(:index_exists?).with(index_name: /gitlab-test-/).and_return(matcher)
expect(subject.completed?).to eq(matcher)
end
end
end
end
RSpec.shared_examples 'a deprecated Advanced Search migration' do |version|
subject { described_class.new(version) }
......
......@@ -51,14 +51,32 @@
end
describe '#es_parent' do
let(:target) { create(:merge_request) }
context 'when target type is in routing excluded list' do
let(:target) { project }
it 'includes project id' do
expect(subject.es_parent).to eq("project_#{target.project.id}")
it 'is nil' do
expect(subject.es_parent).to be_nil
end
end
context 'when target type is in routing excluded list' do
let(:target) { project }
context 'when target has a project_id' do
let(:target) { create(:merge_request) }
it 'includes project id' do
expect(subject.es_parent).to eq("project_#{target.project.id}")
end
end
context 'when target has a group_id' do
let(:target) { create(:epic) }
it 'includes group id' do
expect(subject.es_parent).to eq("group_#{target.group.id}")
end
end
context 'when target does not have project_id or group_id' do
let(:target) { create(:user) }
it 'is nil' do
expect(subject.es_parent).to be_nil
......
# frozen_string_literal: true
require 'spec_helper'
require_relative './config_shared_examples'
RSpec.describe Elastic::Latest::EpicConfig, feature_category: :global_search do
describe '.settings' do
it_behaves_like 'config settings return correct values'
end
describe '.mappings' do
it 'returns config' do
expect(described_class.mapping).to be_a(Elasticsearch::Model::Indexing::Mappings)
end
end
describe '.index_name' do
it 'includes' do
expect(described_class.index_name).to include('-epics')
end
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment