Commit 07e1bcc0 authored by Yorick Peterse's avatar Yorick Peterse

Merge branch 'bvl-fix-concurrent-fork-network-migrations' into 'master'

Fix concurrent fork network migrations

Closes #41487

See merge request !16988
parents 297b075b 917fb174
Pipeline #17259431 passed with stages
in 24 minutes and 22 seconds
---
title: Avoid running `PopulateForkNetworksRange`-migration multiple times
merge_request: 16988
author:
type: fixed
...@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration ...@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
MIGRATION = 'PopulateForkNetworksRange'.freeze
BATCH_SIZE = 100
DELAY_INTERVAL = 15.seconds
disable_ddl_transaction!
class ForkedProjectLink < ActiveRecord::Base
include EachBatch
self.table_name = 'forked_project_links'
end
def up def up
say 'Populating the `fork_networks` based on existing `forked_project_links`' say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
end end
def down def down
......
...@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration ...@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
MIGRATION = 'PopulateForkNetworksRange'.freeze
BATCH_SIZE = 100
DELAY_INTERVAL = 15.seconds
disable_ddl_transaction!
class ForkedProjectLink < ActiveRecord::Base
include EachBatch
self.table_name = 'forked_project_links'
end
def up def up
say 'Populating the `fork_networks` based on existing `forked_project_links`' say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
end end
def down def down
......
...@@ -94,6 +94,18 @@ jobs = [['BackgroundMigrationClassName', [1]], ...@@ -94,6 +94,18 @@ jobs = [['BackgroundMigrationClassName', [1]],
BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs) BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs)
``` ```
### Rescheduling background migrations
If one of the background migrations contains a bug that is fixed in a patch
release, the background migration needs to be rescheduled so the migration would
be repeated on systems that already performed the initial migration.
When you reschedule the background migration, make sure to turn the original
scheduling into a no-op by clearing up the `#up` and `#down` methods of the
migration performing the scheduling. Otherwise the background migration would be
scheduled multiple times on systems that are upgrading multiple patch releases at
once.
## Cleaning Up ## Cleaning Up
>**Note:** >**Note:**
......
...@@ -14,6 +14,14 @@ module Gitlab ...@@ -14,6 +14,14 @@ module Gitlab
def perform(start_id, end_id) def perform(start_id, end_id)
log("Creating memberships for forks: #{start_id} - #{end_id}") log("Creating memberships for forks: #{start_id} - #{end_id}")
insert_members(start_id, end_id)
if missing_members?(start_id, end_id)
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id])
end
end
def insert_members(start_id, end_id)
ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS
INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id)
...@@ -33,10 +41,9 @@ module Gitlab ...@@ -33,10 +41,9 @@ module Gitlab
WHERE existing_members.project_id = forked_project_links.forked_to_project_id WHERE existing_members.project_id = forked_project_links.forked_to_project_id
) )
INSERT_MEMBERS INSERT_MEMBERS
rescue ActiveRecord::RecordNotUnique => e
if missing_members?(start_id, end_id) # `fork_network_member` was created concurrently in another migration
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) log(e.message)
end
end end
def missing_members?(start_id, end_id) def missing_members?(start_id, end_id)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment