Skip to content
Snippets Groups Projects
Commit 86527290 authored by James Fargher's avatar James Fargher
Browse files

Wrap rails code in concurrent backups

Rails needs these extra blocks to prevent internal race conditions
parent f2c27366
No related branches found
No related tags found
1 merge request!39894Fix race condition in concurrent backups
---
title: Fix race condition in concurrent backups
merge_request: 39894
author:
type: fixed
......@@ -26,13 +26,17 @@ def dump(max_concurrency:, max_storage_concurrency:)
threads = Gitlab.config.repositories.storages.keys.map do |storage|
Thread.new do
dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
rescue => e
errors << e
Rails.application.executor.wrap do
dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
rescue => e
errors << e
end
end
end
threads.each(&:join)
ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
threads.each(&:join)
end
raise errors.pop unless errors.empty?
end
......@@ -155,16 +159,18 @@ def dump_storage(storage, semaphore, max_storage_concurrency:)
threads = Array.new(max_storage_concurrency) do
Thread.new do
while project = queue.pop
semaphore.acquire
begin
dump_project(project)
rescue => e
errors << e
break
ensure
semaphore.release
Rails.application.executor.wrap do
while project = queue.pop
semaphore.acquire
begin
dump_project(project)
rescue => e
errors << e
break
ensure
semaphore.release
end
end
end
end
......@@ -177,7 +183,9 @@ def dump_storage(storage, semaphore, max_storage_concurrency:)
end
queue.close
threads.each(&:join)
ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
threads.each(&:join)
end
raise errors.pop unless errors.empty?
end
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment