Skip to content
Snippets Groups Projects

Add lease to update project statistics row and log concurrent updates

Merged Albert requested to merge 373595-lock-on-project-statistics-update into master
4 files
+ 104
101
Compare changes
  • Side-by-side
  • Inline
Files
4
@@ -47,6 +47,7 @@ module CounterAttribute
WORKER_DELAY = 10.minutes
WORKER_LOCK_TTL = 10.minutes
DATABASE_LOCK_RETRY_INTERVAL = 0.5
class_methods do
def counter_attribute(attribute)
@@ -96,7+97,7 @@
transaction do
update_counters_with_lock({ attribute => increment_value })
redis_state { |redis| redis.del(flushed_key) }
new_db_value = reset.read_attribute(attribute)
end
@@ -130,7+131,7 @@
end
end
def update_counters_with_lock(increments)
with_lock_for_update(log_fields: increments.merge({ caller: __method__ })) do
self.class.update_counters(id, increments)
end
end
def reset_counter!(attribute)
try_obtain_lock(log_fields: { caller: __method__ }) do
with_lock_for_update(log_fields: { caller: __method__ }) do
update!(attribute => 0)
clear_counter!(attribute)
end
@@ -169,0+176,0 @@
self.class.counter_attribute_enabled?(attribute)
end
def with_lock_for_update(log_fields: {})
Retriable.retriable(
tries: 3,
base_interval: DATABASE_LOCK_RETRY_INTERVAL,
on: ActiveRecord::LockWaitTimeout,
on_retry: retry_logger(log_fields)
) do
with_lock('FOR UPDATE NOWAIT') do
yield
end
end
end
private
def steal_increments(increment_key, flushed_key)
@@ -230,4 +250,16 @@ def log_clear_counter(attribute)
Gitlab::AppLogger.info(payload)
end
def retry_logger(log_fields = {})
return proc {} unless Feature.enabled?(:warn_concurrent_project_statistics_update, type: :ops)
proc do
Gitlab::AppLogger.warn(
message: 'Concurrent update to project statistics detected',
project_statistics_id: id,
**log_fields
)
end
end
end
Loading