Skip to content
Snippets Groups Projects
Commit 70d7be54 authored by Madelein van Niekerk's avatar Madelein van Niekerk :one:
Browse files

Merge branch '250699-epic-index' into 'master'

Draft: Create Epic Elasticsearch index

See merge request gitlab-org/gitlab!121635



Merged-by: default avatarMadelein van Niekerk <mvanniekerk@gitlab.com>
parents 57cae517 1c68cb26
No related branches found
No related tags found
No related merge requests found
Showing
with 246 additions and 4 deletions
......@@ -92,6 +92,9 @@ Search/NamespacedClass:
- 'ee/lib/elastic/latest/config.rb'
- 'ee/lib/elastic/latest/custom_language_analyzers.rb'
- 'ee/lib/elastic/latest/document_should_be_deleted_from_index_error.rb'
- 'ee/lib/elastic/latest/epic_class_proxy.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/epic_instance_proxy.rb'
- 'ee/lib/elastic/latest/git_class_proxy.rb'
- 'ee/lib/elastic/latest/git_instance_proxy.rb'
- 'ee/lib/elastic/latest/issue_class_proxy.rb'
......@@ -131,6 +134,8 @@ Search/NamespacedClass:
- 'ee/lib/elastic/v12p1/application_class_proxy.rb'
- 'ee/lib/elastic/v12p1/application_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/config.rb'
- 'ee/lib/elastic/v12p1/epic_class_proxy.rb'
- 'ee/lib/elastic/v12p1/epic_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_class_proxy.rb'
- 'ee/lib/elastic/v12p1/issue_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/merge_request_class_proxy.rb'
......
......@@ -233,6 +233,7 @@ Style/RedundantSelf:
- 'ee/lib/ee/model.rb'
- 'ee/lib/elastic/instance_proxy_util.rb'
- 'ee/lib/elastic/latest/commit_config.rb'
- 'ee/lib/elastic/latest/epic_config.rb'
- 'ee/lib/elastic/latest/issue_config.rb'
- 'ee/lib/elastic/latest/merge_request_config.rb'
- 'ee/lib/elastic/latest/note_config.rb'
......
---
name: search_index_curation_epics
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/413605
milestone: '16.1'
type: ops
group: group::global search
default_enabled: false
......@@ -23,6 +23,7 @@ module Epic
include EachBatch
include ::Exportable
include Epics::MetadataCacheUpdate
include Elastic::ApplicationVersionedSearch
DEFAULT_COLOR = ::Gitlab::Color.of('#1068bf')
MAX_HIERARCHY_DEPTH = 7
......@@ -325,6 +326,17 @@ def search(query)
fuzzy_search(query, [:title, :description])
end
def elasticsearch_available?
return false unless ::Feature.enabled?(:elastic_index_epics)
::Elastic::DataMigrationService.migration_has_finished?(:create_epic_index)
end
override :use_separate_indices?
def use_separate_indices?
true
end
def ids_for_base_and_decendants(epic_ids)
::Gitlab::ObjectHierarchy.new(self.id_in(epic_ids)).base_and_descendants.pluck(:id)
end
......@@ -519,6 +531,12 @@ def validate_parent
validate_parent_epic
end
def use_elasticsearch?
return false unless self.class.elasticsearch_available?
group&.use_elasticsearch?
end
def issues_readable_by(current_user, preload: nil)
related_issues = self.class.related_issues(ids: id, preload: preload)
......
......@@ -27,7 +27,7 @@ module Group
add_authentication_token_field :saml_discovery_token, unique: false, token_generator: -> { Devise.friendly_token(8) }
has_many :epics
has_many :epics, dependent: :destroy
has_many :epic_boards, class_name: 'Boards::EpicBoard', inverse_of: :group
has_many :iterations
has_many :iterations_cadences, class_name: 'Iterations::Cadence'
......
......@@ -105,6 +105,21 @@ def maintain_indexed_associations(object, associations)
end
end
def maintain_indexed_group_associations!(*groups, ignore_use_elasticsearch: false)
group_associations = Epic.elasticsearch_available? ? [:epics] : []
return unless group_associations.any?
groups.each do |group|
raise ArgumentError, 'This method only accepts Groups' unless group.is_a?(Group)
next if !ignore_use_elasticsearch && !group.use_elasticsearch?
ElasticAssociationIndexerWorker
.perform_async(group.class.name, group.id, group_associations, ignore_use_elasticsearch)
end
end
private
def each_indexed_association(object, associations)
......
......@@ -2,6 +2,7 @@
module Elastic
class ProcessInitialBookkeepingService < Elastic::ProcessBookkeepingService
INDEXED_GROUP_ASSOCIATIONS = [:epics].freeze
INDEXED_PROJECT_ASSOCIATIONS = [
:issues,
:merge_requests,
......
......@@ -12,6 +12,7 @@ def perform(id)
namespace = Namespace.find(id)
update_users_through_membership(namespace)
update_epics(namespace)
end
def update_users_through_membership(namespace)
......@@ -31,6 +32,10 @@ def update_users_through_membership(namespace)
# rubocop:enable CodeReuse/ActiveRecord
end
def update_epics(namespace)
Elastic::ProcessBookkeepingService.maintain_indexed_group_associations!(namespace)
end
def group_and_descendants_user_ids(namespace)
namespace.self_and_descendants.flat_map(&:user_ids)
end
......
......@@ -11,12 +11,12 @@ class ElasticAssociationIndexerWorker # rubocop:disable Scalability/IdempotentWo
worker_resource_boundary :cpu
loggable_arguments 0, 2
def perform(class_name, id, indexed_associations)
def perform(class_name, id, indexed_associations, ignore_use_elasticsearch = false)
return unless Gitlab::CurrentSettings.elasticsearch_indexing?
klass = class_name.constantize
object = klass.find(id)
return unless object.use_elasticsearch?
return if !ignore_use_elasticsearch && !object.use_elasticsearch?
Elastic::ProcessBookkeepingService.maintain_indexed_associations(object, indexed_associations)
end
......
......@@ -19,6 +19,7 @@ def perform(namespace_id, operation)
case operation.to_s
when /index/
index_projects(namespace)
index_group_associations(namespace)
when /delete/
delete_from_index(namespace)
end
......@@ -32,10 +33,21 @@ def index_projects(namespace)
end
end
def index_group_associations(namespace)
return unless namespace.group_namespace?
Elastic::ProcessBookkeepingService.maintain_indexed_group_associations!(namespace)
end
def delete_from_index(namespace)
namespace.all_projects.find_in_batches do |batch|
args = batch.map { |project| [project.id, project.es_id] }
ElasticDeleteProjectWorker.bulk_perform_async(args) # rubocop:disable Scalability/BulkPerformWithContext
end
return unless namespace.group_namespace?
Elastic::ProcessBookkeepingService
.maintain_indexed_group_associations!(*namespace.self_and_descendants, ignore_use_elasticsearch: true)
end
end
---
name: elastic_index_epics
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/412108
milestone: '16.1'
type: development
group: group::global search
default_enabled: false
# frozen_string_literal: true
class CreateEpicIndex < Elastic::Migration
include Elastic::MigrationCreateIndex
retry_on_failure
def document_type
:epic
end
def target_class
Epic
end
end
......@@ -6,7 +6,9 @@ class ApplicationInstanceProxy < Elasticsearch::Model::Proxy::InstanceMethodsPro
include InstanceProxyUtil
def es_parent
"project_#{target.project_id}" unless target.is_a?(Project) || target&.project_id.nil?
return if target.is_a?(Project)
return "project_#{target.project_id}" if target.respond_to?(:project_id) && target.project_id.present?
return "group_#{target.group_id}" if target.respond_to?(:group_id) && target.group_id.present?
end
def es_type
......
# frozen_string_literal: true
module Elastic
module Latest
class EpicClassProxy < ApplicationClassProxy
def elastic_search(query, options: {})
raise NotImplementedError
end
def preload_indexing_data(relation)
# rubocop: disable CodeReuse/ActiveRecord
relation.includes(
:author,
:labels,
:group,
:start_date_sourcing_epic,
:due_date_sourcing_epic,
:start_date_sourcing_milestone,
:due_date_sourcing_milestone
)
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
module EpicConfig
extend Elasticsearch::Model::Indexing::ClassMethods
extend Elasticsearch::Model::Naming::ClassMethods
self.index_name = [Rails.application.class.module_parent_name.downcase, Rails.env, 'epics'].join('-')
settings Elastic::Latest::Config.settings.to_hash.deep_merge(
index: {
number_of_shards: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_shards },
number_of_replicas: Elastic::AsJSON.new { Elastic::IndexSetting[self.index_name].number_of_replicas }
}
)
mappings dynamic: 'strict' do
indexes :id, type: :integer
indexes :iid, type: :integer
indexes :group_id, type: :integer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :title, type: :text, index_options: 'positions'
indexes :description, type: :text, index_options: 'positions'
indexes :state, type: :keyword
indexes :confidential, type: :boolean
indexes :author_id, type: :integer
indexes :label_ids, type: :keyword
indexes :start_date, type: :date
indexes :due_date, type: :date
indexes :traversal_ids, type: :keyword
indexes :hashed_root_namespace_id, type: :integer
indexes :schema_version, type: :short
indexes :type, type: :keyword
end
end
end
end
# frozen_string_literal: true
module Elastic
module Latest
class EpicInstanceProxy < ApplicationInstanceProxy
def as_indexed_json(_options = {})
data = {}
[
:id,
:iid,
:group_id,
:created_at,
:updated_at,
:title,
:description,
:state,
:confidential,
:author_id
].each do |attr|
data[attr.to_s] = safely_read_attribute_for_elasticsearch(attr)
end
data['label_ids'] = target.label_ids.map(&:to_s)
data['start_date'] = target.start_date || target.start_date_from_inherited_source
data['due_date'] = target.end_date || target.due_date_from_inherited_source
data['traversal_ids'] = target.group.elastic_namespace_ancestry
data['hashed_root_namespace_id'] = target.group.hashed_root_namespace_id
# Schema version. The format is Date.today.strftime('%y_%m')
# Please update if you're changing the schema of the document
data['schema_version'] = 23_05
data.merge(generic_attributes)
end
def generic_attributes
super.except('join_field')
end
end
end
end
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicClassProxy = Elastic::Latest::EpicClassProxy
end
end
# rubocop:enable Naming/FileName
# rubocop:disable Naming/FileName
# frozen_string_literal: true
module Elastic
module V12p1
EpicInstanceProxy = Elastic::Latest::EpicInstanceProxy
end
end
# rubocop:enable Naming/FileName
......@@ -11,6 +11,7 @@ class Helper
Milestone,
ProjectWiki,
Repository,
Epic,
User
].freeze
......@@ -19,6 +20,7 @@ class Helper
Note,
MergeRequest,
Commit,
Epic,
User,
Wiki
].freeze
......
......@@ -25,6 +25,7 @@ namespace :gitlab do
Rake::Task["gitlab:elastic:index_projects"].invoke
Rake::Task["gitlab:elastic:index_snippets"].invoke
Rake::Task["gitlab:elastic:index_users"].invoke
Rake::Task["gitlab:elastic:index_epics"].invoke
end
desc 'GitLab | Elasticsearch | Enable Elasticsearch search'
......@@ -101,6 +102,24 @@ namespace :gitlab do
logger.info("Indexing users... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Index epics"
task index_epics: :environment do
logger = Logger.new($stdout)
logger.info("Indexing epics...")
groups = Group.all
if ::Gitlab::CurrentSettings.elasticsearch_limit_indexing?
groups.merge!(::Gitlab::CurrentSettings.elasticsearch_limited_namespaces)
end
groups.each_batch do |batch|
::Elastic::ProcessInitialBookkeepingService.backfill_group_associations!(*batch)
end
logger.info("Indexing epics... " + "done".color(:green))
end
desc "GitLab | Elasticsearch | Create empty indexes and assigns an alias for each"
task create_empty_index: [:environment] do |t, args|
with_alias = ENV["SKIP_ALIAS"].nil?
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment