Commit c444bbc1 authored by Alexandru Croitor's avatar Alexandru Croitor

Enqueue background jobs to run async in batches

Improve migration to run a single update statement per batch,
and update multiple values in one statement.
parent 9b351619
Pipeline #71903260 passed with stages
in 69 minutes and 48 seconds
......@@ -7,10 +7,37 @@ class MigrateDiscussionIdOnPromotedEpics < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
BATCH_SIZE = 100
BATCH_SIZE = 500
DELAY_INTERVAL = 2.minutes
MIGRATION = 'MigratePromotedEpicsDiscussionIds'
disable_ddl_transaction!
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
def self.fetch_discussion_ids_query
promoted_epics_query = Note
.where(system: true)
.where(noteable_type: 'Epic')
.where("note LIKE 'promoted from%'")
.select("DISTINCT noteable_id")
Note.where(noteable_type: 'Epic')
.where(noteable_id: promoted_epics_query)
.select("DISTINCT discussion_id").order(:discussion_id)
end
end
def up
Gitlab::BackgroundMigration::MigratePromotedEpicsDiscussionIds.new.perform_all_sync(batch_size: BATCH_SIZE)
Note.fetch_discussion_ids_query.each_batch(of: BATCH_SIZE, column: :discussion_id) do |relation, index|
discussion_ids = relation.collect(&:discussion_id)
delay = index * DELAY_INTERVAL
BackgroundMigrationWorker.perform_in(delay, MIGRATION, [discussion_ids])
end
end
def down
# no-op
end
end
---
title: Migrate promoted epics discussion id
merge_request:
merge_request: 14708
author:
type: fixed
......@@ -6,35 +6,25 @@ module Gitlab
# is different from discussion id on issue, which was causing problems when repying to epic discussions as it would
# identify the discussion as related to an issue and complaint about missing project_id
class MigratePromotedEpicsDiscussionIds
# notes model to itterate through the notes to be updated
# notes model to iterate through the notes to be updated
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
end
def perform(discussion_id)
Note.where(noteable_type: 'Epic').where(discussion_id: discussion_id).update_all(discussion_id: Discussion.discussion_id(Note.new))
end
def perform_all_sync(batch_size:)
fetch_discussion_ids_query.each_batch(of: batch_size) do |notes|
notes.each do |note|
perform(note[:discussion_id])
end
end
end
private
def perform(discussion_ids)
new_discussion_ids = discussion_ids.map {|old_id| [old_id, Discussion.discussion_id(Note.new)]}
values = new_discussion_ids.map { |el| el.map { |id| Note.connection.quote_string(id) }.join("','") }.join("'), ('").prepend("('").concat("')")
sql = <<-SQL.squish
UPDATE notes SET discussion_id = v.new_discussion_id
FROM (
VALUES
#{values}
) AS v(old_discussion_id, new_discussion_id)
WHERE notes.discussion_id = v.old_discussion_id
AND notes.noteable_type = 'Epic'
SQL
def fetch_discussion_ids_query
promoted_epics_query = Note
.where(system: true)
.where(noteable_type: 'Epic')
.where("note LIKE 'promoted from%'")
.select("DISTINCT noteable_id")
Note.where(noteable_type: 'Epic')
.where(noteable_id: promoted_epics_query)
.select("DISTINCT discussion_id")
Note.connection.execute(sql)
end
end
end
......
......@@ -19,43 +19,15 @@ describe Gitlab::BackgroundMigration::MigratePromotedEpicsDiscussionIds, :migrat
let!(:epic1_note1) { notes.create(note: 'note comment', noteable_id: epic1.id, noteable_type: 'Epic', discussion_id: 'd1') }
let!(:epic1_note2) { notes.create(system: true, note: 'promoted from issue XXX', noteable_id: epic1.id, noteable_type: 'Epic', discussion_id: 'system1') }
def create_merge_request(id, params = {})
params.merge!(id: id,
target_project_id: project.id,
target_branch: 'master',
source_project_id: project.id,
source_branch: 'mr name',
title: "mr name#{id}")
merge_requests.create(params)
end
describe '#perform' do
it 'updates epic note discussion id to a newlly generated discussion id' do
expect(notes.where(discussion_id: 'd1').count).to eq(2)
subject.perform(epic1_note1.discussion_id)
expect(notes.where(discussion_id: 'd1').count).to eq(1)
expect(notes.where(discussion_id: 'd1').first.noteable_type).to eq('Issue')
expect(epic1_note1.reload.discussion_id).not_to eq('d1')
end
end
describe '#perform_all_sync' do
describe '#perform with batch of discussion ids' do
let(:epic2) { epics.create(id: 2, author_id: user.id, iid: 2, group_id: namespace.id, title: 'Epic with discussion', title_html: 'Epic with discussion') }
let!(:epic2_note1) { notes.create(note: 'note comment', noteable_id: epic2.id, noteable_type: 'Epic', discussion_id: 'd2') }
let!(:epic2_note2) { notes.create(system: true, note: 'promoted from issue YYY', noteable_id: epic2.id, noteable_type: 'Epic', discussion_id: 'system2') }
it 'executes perform for all discussions on all promoted epics' do
expect(subject).to receive(:perform).exactly(4).times # d1, system1, d2, system2
subject.perform_all_sync(batch_size: 3)
end
it 'executes peform and changes discussion ids on all promoted epic discussions' do
it 'executes perform and changes discussion ids on all promoted epic discussions' do
expect(notes.where(discussion_id: 'd1').count).to eq(2)
subject.perform_all_sync(batch_size: 3)
subject.perform(%w(d1 system1 d2 system2))
expect(notes.where(discussion_id: 'd1').count).to eq(1)
expect(notes.where(discussion_id: 'd1').first.noteable_type).to eq('Issue')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment