Commit 3ef4154f authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre 🌴

Avoid backfill worker to stuck when clone take longer than run time

[ci skip]
parent 0cf48d8f
module Geo
class RepositoryBackfillService
attr_reader :project
attr_reader :project, :backfill_lease
LEASE_TIMEOUT = 8.hours.freeze
LEASE_TIMEOUT = 8.hours.freeze
LEASE_KEY_PREFIX = 'repository_backfill_service'.freeze
def initialize(project)
@project = project
def initialize(project_id, backfill_lease)
@project = Project.find(project_id)
@backfill_lease = backfill_lease
end
def execute
......@@ -16,7 +18,6 @@ module Geo
registry.last_repository_synced_at = started_at
registry.last_repository_successful_sync_at = finished_at if finished_at
registry.save
log('Finished repository sync')
end
end
......@@ -58,22 +59,19 @@ module Geo
def try_obtain_lease
log('Trying to obtain lease to sync repository')
uuid = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
repository_lease = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
log('Could not obtain lease to sync repository') and return unless uuid
log('Could not obtain lease to sync repository') and return unless repository_lease
yield
log('Releasing lease to sync repository')
release_lease(uuid)
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
log('Releasing leases to sync repository')
Gitlab::ExclusiveLease.cancel(lease_key, repository_lease)
Gitlab::ExclusiveLease.cancel(LEASE_KEY_PREFIX, backfill_lease)
end
def lease_key
@key ||= "repository_backfill_service:#{project.id}"
@key ||= "#{LEASE_KEY_PREFIX}:#{project.id}"
end
def primary_ssh_path_prefix
......
......@@ -2,8 +2,7 @@ class GeoBackfillWorker
include Sidekiq::Worker
include CronjobQueue
LEASE_TIMEOUT = 8.hours.freeze
RUN_TIME = 5.minutes.to_i.freeze
RUN_TIME = 5.minutes.to_i.freeze
def perform
start = Time.now
......@@ -18,8 +17,8 @@ class GeoBackfillWorker
project = Project.find(project_id)
next if project.repository_exists?
try_obtain_lease do
Geo::RepositoryBackfillService.new(project).execute
try_obtain_lease do |lease|
GeoSingleRepositoryBackfillWorker.new.perform(project_id, lease)
end
end
......@@ -37,21 +36,19 @@ class GeoBackfillWorker
end
def try_obtain_lease
uuid = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
lease = Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT).try_obtain
return unless uuid
return unless lease
yield
release_lease(uuid)
yield lease
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
def lease_key
Geo::RepositoryBackfillService::LEASE_KEY_PREFIX
end
def lease_key
'repository_backfill_service'
def lease_timeout
Geo::RepositoryBackfillService::LEASE_TIMEOUT
end
def node_enabled?
......
class GeoSingleRepositoryBackfillWorker
include Sidekiq::Worker
include ::GeoDynamicBackoff
include GeoQueue
def perform(project_id, lease)
Geo::RepositoryBackfillService.new(project_id, lease).execute
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment