Skip to content
Snippets Groups Projects
Verified Commit ad40b0c8 authored by Siddharth Dungarwal's avatar Siddharth Dungarwal :two: Committed by GitLab
Browse files

Archive create_work_items_index migration

parent 3585345e
No related branches found
No related tags found
1 merge request!170781Archive create_work_items_index migration
Showing
with 64 additions and 239 deletions
......@@ -32,19 +32,10 @@ def maintain_elasticsearch_destroy
private
def work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
def get_indexing_data
indexing_data = [self]
# We can remove issue_id check too when we are cleaning the migration completion check
if work_item_index_available? && issue_id
indexing_data << Search::Elastic::References::WorkItem.new(issue_id,
"group_#{group.root_ancestor.id}")
end
indexing_data << Search::Elastic::References::WorkItem.new(
issue_id, "group_#{group.root_ancestor.id}")
indexing_data
end
end
......
......@@ -56,25 +56,18 @@ def indexing_issue_of_epic_type?
project.nil?
end
def work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
def get_indexing_data
indexing_data = []
case self
when WorkItem
indexing_data << self if work_item_index_available?
indexing_data << self
unless indexing_issue_of_epic_type?
indexing_data << Search::Elastic::References::Legacy.instantiate_from_array([Issue, id, es_id,
"project_#{project.id}"])
end
when Issue
if work_item_index_available?
indexing_data << Search::Elastic::References::WorkItem.new(id, "group_#{namespace.root_ancestor.id}")
end
indexing_data << Search::Elastic::References::WorkItem.new(id, "group_#{namespace.root_ancestor.id}")
indexing_data << self unless indexing_issue_of_epic_type?
end
indexing_data << synced_epic if synced_epic.present?
......
......@@ -72,8 +72,6 @@ def skip_metrics?
override :use_elasticsearch?
def use_elasticsearch?
return super unless ::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
namespace.use_elasticsearch?
end
......
......@@ -28,13 +28,7 @@ def logger
@logger ||= ::Gitlab::Elasticsearch::Logger.build
end
def work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
def remove_work_item_documents(project_id, traversal_id)
return unless work_item_index_available?
filter_list = []
filter_list << { term: { project_id: project_id } } unless project_id.nil?
......
......@@ -169,17 +169,11 @@ def resume_indexing
end
end
def work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
def estimate_shard_sizes
estimates = {}
klasses = CLASSES_TO_COUNT
klasses -= [WorkItem] unless work_item_index_available?
counts = ::Gitlab::Database::Count.approximate_counts(klasses)
klasses.each do |klass|
......@@ -376,8 +370,6 @@ def index_group_entities
end
def index_work_items
return unless work_item_index_available?
logger.info('Indexing work_items...')
work_items = if ::Gitlab::CurrentSettings.elasticsearch_limit_indexing?
......
......@@ -19,21 +19,16 @@ def perform(group_id, ancestor_id, options = {})
group = Group.find_by_id(group_id)
remove_epics = index_epics?(group)
remove_work_items = work_item_index_available?
return unless remove_work_items || remove_epics
options = options.with_indifferent_access
unless options[:include_descendants]
return process_removal(group_id, ancestor_id, remove_epics: remove_epics, remove_work_items: remove_work_items)
end
return process_removal(group_id, ancestor_id, remove_epics: remove_epics) unless options[:include_descendants]
# We have the return condition here because we still want to remove the deleted items in the above call
return if group.nil?
# rubocop: disable CodeReuse/ActiveRecord -- We need only the ids of self_and_descendants groups
group.self_and_descendants.each_batch do |groups|
process_removal(groups.pluck(:id), ancestor_id, remove_epics: remove_epics,
remove_work_items: remove_work_items)
process_removal(groups.pluck(:id), ancestor_id, remove_epics: remove_epics)
end
# rubocop: enable CodeReuse/ActiveRecord
end
......@@ -44,21 +39,15 @@ def client
@client ||= ::Gitlab::Search::Client.new
end
def work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
def index_epics?(group)
return true unless group.present?
group.licensed_feature_available?(:epics)
end
def process_removal(group_id, ancestor_id, remove_epics:, remove_work_items:)
if remove_work_items
remove_items(group_id, ancestor_id, index_name: ::Search::Elastic::Types::WorkItem.index_name,
group_id_field: :namespace_id)
end
def process_removal(group_id, ancestor_id, remove_epics:)
remove_items(group_id, ancestor_id, index_name: ::Search::Elastic::Types::WorkItem.index_name,
group_id_field: :namespace_id)
return unless remove_epics
......
......@@ -5,6 +5,6 @@ description: Add a new work items index
group: group::global search
milestone: '17.1'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/151636
obsolete: false
marked_obsolete_by_url:
marked_obsolete_in_milestone:
obsolete: true
marked_obsolete_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/170781
marked_obsolete_in_milestone: '17.6'
......@@ -13,3 +13,5 @@ def target_class
WorkItem
end
end
CreateWorkItemsIndex.prepend ::Elastic::MigrationObsolete
......@@ -17,16 +17,10 @@ def self.serialize(record)
def self.instantiate(string)
_, id, routing = delimit(string)
return unless work_item_index_available?
# this puts the record in the work items index
new(id, routing)
end
def self.work_item_index_available?
::Elastic::DataMigrationService.migration_has_finished?(:create_work_items_index)
end
override :preload_indexing_data
def self.preload_indexing_data(refs)
ids = refs.map(&:identifier)
......
......@@ -4,5 +4,5 @@
require File.expand_path('ee/elastic/migrate/20240501134252_create_work_items_index.rb')
RSpec.describe CreateWorkItemsIndex, feature_category: :global_search do
it_behaves_like 'migration creates a new index', 20240501134252, WorkItem
it_behaves_like 'a deprecated Advanced Search migration', 20240501134252
end
......@@ -28,18 +28,6 @@
context 'when an epic is created' do
let(:epic) { build(:epic, group: group) }
context 'when create_work_items_index migration is not complete' do
before do
allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:create_work_items_index).and_return(false)
end
it 'tracks the epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
epic.save!
end
end
it 'tracks the epic and work item' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(2)
......@@ -90,18 +78,6 @@
epic.destroy!
end
context 'when create_work_items_index migration is not complete' do
before do
allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:create_work_items_index).and_return(false)
end
it 'tracks the epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
epic.destroy!
end
end
it 'deletes the epic from elasticsearch', :elastic_clean do
allow(::Elastic::ProcessBookkeepingService).to receive(:track!).and_call_original
......
......@@ -280,26 +280,10 @@
describe '#instantiate' do
let(:work_item_ref) { described_class.new(work_item.id, work_item.es_parent) }
context 'when work_item index is available' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: true
end
it 'instantiates work item' do
new_work_item = described_class.instantiate(work_item_ref.serialize)
expect(new_work_item.routing).to eq(work_item.es_parent)
expect(new_work_item.identifier).to eq(work_item.id)
end
end
context 'when migration is not completed' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: false
end
it 'does not instantiate work item' do
expect(described_class.instantiate(work_item_ref.serialize)).to be_nil
end
it 'instantiates work item' do
new_work_item = described_class.instantiate(work_item_ref.serialize)
expect(new_work_item.routing).to eq(work_item.es_parent)
expect(new_work_item.identifier).to eq(work_item.id)
end
end
......
......@@ -13,20 +13,18 @@
before do
issue_epic_type.project = nil # Need to set this to nil as :epic feature is not enforing it.
allow(Gitlab::Elastic::Helper).to receive(:default).and_return(helper)
# Enforcing this to false because we have test for truthy in work_item_index_spec.rb
set_elasticsearch_migration_to :create_work_items_index, including: false
allow(Gitlab::Saas).to receive(:feature_available?).with(:ai_vertex_embeddings).and_return(false)
end
describe '#maintain_elasticsearch_update' do
it 'calls track! for non group level WorkItem' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
expect(tracked_refs[0].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[0].klass).to eq(Issue)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(WorkItem)
expect(tracked_refs[0].id).to eq(non_group_work_item.id)
expect(tracked_refs[1]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
expect(tracked_refs[1].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[1].klass).to eq(Issue)
end
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
......@@ -39,22 +37,26 @@
it 'calls track! for group level Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(0)
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0].identifier).to eq(issue_epic_type.id)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
end
issue_epic_type.maintain_elasticsearch_update
end
it 'calls track! for synced_epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item.synced_epic])
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item, work_item.synced_epic])
work_item.maintain_elasticsearch_update
end
it 'calls track! with Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(Issue)
expect(tracked_refs[0].id).to eq(issue.id)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
expect(tracked_refs[0].identifier).to eq(issue.id)
expect(tracked_refs[1]).to be_a_kind_of(Issue)
expect(tracked_refs[1].id).to eq(issue.id)
end
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
......@@ -197,10 +199,12 @@
describe '#maintain_elasticsearch_destroy' do
it 'calls track! for non group level WorkItem' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
expect(tracked_refs[0].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[0].klass).to eq(Issue)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(WorkItem)
expect(tracked_refs[0].id).to eq(non_group_work_item.id)
expect(tracked_refs[1].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[1].klass).to eq(Issue)
expect(tracked_refs[1]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
end
non_group_work_item.maintain_elasticsearch_destroy
......@@ -208,22 +212,26 @@
it 'calls track! for group level Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(0)
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0].identifier).to eq(issue_epic_type.id)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
end
issue_epic_type.maintain_elasticsearch_destroy
end
it 'calls track! for synced_epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item.synced_epic])
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item, work_item.synced_epic])
work_item.maintain_elasticsearch_destroy
end
it 'calls track! with Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(Issue)
expect(tracked_refs[0].id).to eq(issue.id)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
expect(tracked_refs[0].identifier).to eq(issue.id)
expect(tracked_refs[1]).to be_a_kind_of(Issue)
expect(tracked_refs[1].id).to eq(issue.id)
end
issue.maintain_elasticsearch_destroy
......@@ -233,32 +241,38 @@
describe '#maintain_elasticsearch_create' do
it 'calls track! for non group level WorkItem' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
expect(tracked_refs[0].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[0].klass).to eq(Issue)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(WorkItem)
expect(tracked_refs[0].id).to eq(non_group_work_item.id)
expect(tracked_refs[1].db_id).to eq(non_group_work_item.id.to_s)
expect(tracked_refs[1].klass).to eq(Issue)
expect(tracked_refs[1]).to be_a_kind_of(::Gitlab::Elastic::DocumentReference)
end
non_group_work_item.maintain_elasticsearch_create
end
it 'calls track! for synced_epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item.synced_epic])
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(*[work_item, work_item.synced_epic])
work_item.maintain_elasticsearch_create
end
it 'calls track! for group level Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(0)
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0].identifier).to eq(issue_epic_type.id)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
end
issue_epic_type.maintain_elasticsearch_create
end
it 'calls track! with Issue' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(1)
expect(tracked_refs[0]).to be_a_kind_of(Issue)
expect(tracked_refs[0].id).to eq(issue.id)
expect(tracked_refs.count).to eq(2)
expect(tracked_refs[0]).to be_a_kind_of(Search::Elastic::References::WorkItem)
expect(tracked_refs[0].identifier).to eq(issue.id)
expect(tracked_refs[1]).to be_a_kind_of(Issue)
expect(tracked_refs[1].id).to eq(issue.id)
end
issue.maintain_elasticsearch_create
......
......@@ -1372,11 +1372,6 @@ def as_item(item)
describe 'ES related specs' do
let_it_be(:epic) { create(:epic, group: group) }
before do
allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:create_work_items_index).and_return(true)
end
context 'when the group has use_elasticsearch? as true' do
before do
allow(group).to receive(:use_elasticsearch?).and_return(true)
......@@ -1392,18 +1387,6 @@ def as_item(item)
.to receive(:elasticsearch_indexing?).and_return(true)
end
context 'when create_work_items_index migration is not complete' do
before do
allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
.with(:create_work_items_index).and_return(false)
end
it 'tracks the epic' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
epic.update!(title: 'A new title')
end
end
it 'calls ::Elastic::ProcessBookkeepingService.track! when the epic is updated' do
expect(::Elastic::ProcessBookkeepingService).to receive(:track!).once do |*tracked_refs|
expect(tracked_refs.count).to eq(2)
......
......@@ -560,19 +560,8 @@
let_it_be(:namespace) { create(:namespace) }
let_it_be(:work_item) { create(:work_item, namespace: namespace) }
context 'when migration is not complete' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: false
end
it 'returns false' do
expect(work_item.use_elasticsearch?).to be_falsey
end
end
context 'when namespace does not use elasticsearch' do
it 'returns false' do
set_elasticsearch_migration_to :create_work_items_index, including: true
stub_ee_application_setting(elasticsearch_indexing: true, elasticsearch_limit_indexing: true)
expect(work_item.use_elasticsearch?).to be_falsey
......@@ -581,7 +570,6 @@
context 'when work_item index is available and namesapce uses elasticsearch' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: true
stub_ee_application_setting(elasticsearch_indexing: true, elasticsearch_limit_indexing: false)
end
......
......@@ -14,10 +14,6 @@
let_it_be(:project) { create(:project, group: group) }
let(:work_item) { create(:work_item, project: project) }
before do
set_elasticsearch_migration_to :create_work_items_index, including: true
end
context 'when Elasticsearch is enabled', :elastic_delete_by_query do
before do
stub_ee_application_setting(elasticsearch_indexing: true)
......@@ -69,25 +65,6 @@
expect(items_in_index(work_item_index).count).to eq(0)
end
end
context 'when migration is not complete' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: false
end
it 'does not remove work items' do
# items are present already
expect(items_in_index(work_item_index)).to include(work_item.id)
expect(items_in_index(work_item_index).count).to eq(1)
execute
es_helper.refresh_index(index_name: work_item_index)
# work items not removed
expect(items_in_index(work_item_index).count).to eq(1)
expect(items_in_index(work_item_index)).to include(work_item.id)
end
end
end
end
end
......@@ -265,21 +265,6 @@
)
end
context 'when work_items are not in a standalone index' do
let(:counts) { [400, 1500, 10_000_000, 50_000_000, 100_000_000, 4_000] }
let(:counted_items) { described_class::CLASSES_TO_COUNT - [WorkItem] }
before do
set_elasticsearch_migration_to :create_work_items_index, including: false
end
it 'does not include work_items index in shard size estimates' do
expect(logger).not_to receive(:info).with(/gitlab-test-work_items/)
service.execute(:estimate_shard_sizes)
end
end
it 'outputs shard size estimates' do
expected_work_items = <<~ESTIMATE
- gitlab-test-work_items:
......@@ -634,16 +619,6 @@
describe '#index_work_items', :elastic do
let_it_be(:work_item) { create(:work_item) }
context 'when work_item index is not available' do
before do
set_elasticsearch_migration_to(:create_work_items_index, including: false)
end
it 'does not call track for work_items' do
expect(Elastic::ProcessInitialBookkeepingService).not_to receive(:track!)
end
end
it 'calls track! for work_items' do
expect(logger).to receive(:info).with(/Indexing work_items/).twice
expect(Elastic::ProcessInitialBookkeepingService).to receive(:track!).with(work_item)
......
......@@ -13,10 +13,6 @@
let(:helper) { Gitlab::Elastic::Helper.default }
let(:client) { helper.client }
before do
set_elasticsearch_migration_to :create_work_items_index, including: true
end
context 'when indexing is paused' do
before do
allow(Elastic::IndexingControl).to receive(:non_cached_pause_indexing?).and_return(true)
......@@ -43,27 +39,6 @@
ensure_elasticsearch_index!
end
context 'when migration is not complete' do
before do
set_elasticsearch_migration_to :create_work_items_index, including: false
end
it 'does not remove work items' do
# items are present already
expect(items_in_index(work_item_index).count).to eq(2)
expect(items_in_index(work_item_index)).to include(group_work_item.id)
expect(items_in_index(work_item_index)).to include(sub_group_work_item.id)
described_class.new.perform(group.id, parent_group.id)
helper.refresh_index(index_name: work_item_index)
# work items not removed
expect(items_in_index(work_item_index).count).to eq(2)
expect(items_in_index(work_item_index)).to include(group_work_item.id)
expect(items_in_index(work_item_index)).to include(sub_group_work_item.id)
end
end
context 'when work_item index is available' do
context 'when we pass include_descendants' do
it 'deletes work items belonging to the group and its descendants' do
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment