Skip to content
Snippets Groups Projects
Commit a3e7e210 authored by Jonas Larsen's avatar Jonas Larsen :two:
Browse files

Add post migration to prevent loss of data

The total counter keys should not expire as that will cause us to lose
data if the keys are not updated continously.

We started expiring keys due to a bug described here:
!152475

This issue only happened on gitlab.com.
parent d4e7510f
No related branches found
No related tags found
2 merge requests!162233Draft: Script to update Topology Service Gem,!153095Ensure all time counters don't have expiry date
# frozen_string_literal: true
class PersistUsageDataKeysThatShouldNotExpire < Gitlab::Database::Migration[2.2]
milestone '17.1'
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :gitlab_main
REDIS_PREFIX = '{event_counters}'
def up
overrides = YAML.safe_load(File.read('lib/gitlab/usage_data_counters/total_counter_redis_key_overrides.yml'))
Gitlab::Redis::SharedState.with do |redis|
# Update all total counter keys
redis.scan_each(match: "#{REDIS_PREFIX}*", count: 10_000) do |key|
redis.persist(key)
end
# Update all keys from total_counter_redis_keys_overrides.yml
overrides.each_value do |legacy_key|
redis.persist(legacy_key)
end
end
end
def down
# no-op
end
end
87bf6622610cfae5363200a8c75dd6875781a1afd10a2ea99cf5ba544fed2ce7
\ No newline at end of file
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe PersistUsageDataKeysThatShouldNotExpire, :migration, :clean_gitlab_redis_cache, feature_category: :service_ping do
def get_expiry_for_key(key)
Gitlab::Redis::SharedState.with { |redis| redis.ttl(key) }
end
shared_examples 'a Redis key does not expire' do |key|
before do
Gitlab::Redis::SharedState.with do |redis|
redis.incr(key)
redis.expire(key, 6.weeks)
end
end
it 'removes the expiry date from the key' do
migrate!
expect(get_expiry_for_key(key)).to eq(-1)
end
end
shared_examples 'a Redis key that expire' do |key|
before do
Gitlab::Redis::SharedState.with do |redis|
redis.incr(key)
redis.expire(key, 6.weeks)
end
end
it 'does not remove the expiry date from the key' do
migrate!
expect(get_expiry_for_key(key)).to be > 0
end
end
describe '#up' do
it_behaves_like 'a Redis key does not expire', '{event_counters}_some_event'
it_behaves_like 'a Redis key does not expire', '{event_counters}_some_other_event'
it_behaves_like 'a Redis key does not expire', 'WEB_IDE_VIEWS_COUNT'
it_behaves_like 'a Redis key that expire', 'some_non_legacy_key_without_the_right_prefix'
it_behaves_like 'a Redis key that expire', '{hll_counters}_project_action-2024-18'
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment