Skip to content
Snippets Groups Projects
Verified Commit 33e0e05d authored by Michał Wielich's avatar Michał Wielich :two:
Browse files

Remove context from HLLRedis counting 

Remove context from HLLRedis counting
parent 3e777e4c
No related branches found
No related tags found
1 merge request!128511Remove context from HLLRedis counting
Hide whitespace changes
Inline Side-by-side
ee/lib/ee/gitlab/usage_data_counters/hll_redis_counter.rb deleted 100644 → 0
+ 0
− 20
View file @ 3e777e4c
# frozen_string_literal: true
module EE
module Gitlab
module UsageDataCounters
module HLLRedisCounter
extend ActiveSupport::Concern
class_methods do
extend ::Gitlab::Utils::Override
override :valid_context_list
def valid_context_list
super + License.all_plans
end
end
end
end
end
end
ee/spec/lib/ee/gitlab/usage_data_counters/hll_redis_counter_spec.rb deleted 100644 → 0
+ 0
− 153
View file @ 3e777e4c
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_shared_state, feature_category: :service_ping do
using RSpec::Parameterized::TableSyntax
let(:entity1) { 'dfb9d2d2-f56c-4c77-8aeb-6cddc4a1f857' }
let(:entity2) { '1dd9afb2-a3ee-4de1-8ae3-a405579c8584' }
let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' }
let(:default_context) { 'default' }
let(:ultimate_context) { 'ultimate' }
let(:gold_context) { 'gold' }
let(:invalid_context) { 'invalid' }
let(:context_event) { 'context_event' }
let(:other_context_event) { 'other_context_event' }
let(:known_events) do
[
{ name: context_event },
{ name: other_context_event }
].map(&:with_indifferent_access)
end
around do |example|
# We need to freeze to a reference time
# because visits are grouped by the week number in the year
# Without freezing the time, the test may behave inconsistently
# depending on which day of the week test is run.
# Monday 6th of June
described_class.clear_memoization(:known_events)
described_class.clear_memoization(:known_events_names)
reference_time = Time.utc(2020, 6, 1)
travel_to(reference_time) { example.run }
described_class.clear_memoization(:known_events)
described_class.clear_memoization(:known_events_names)
end
describe '.known_events' do
it 'uses Gitlab::Usage::MetricDefinition.all' do
expect(Gitlab::Usage::MetricDefinition).to receive(:all).and_call_original
described_class.known_events
end
it 'returns the list of available events' do
allow(Gitlab::Usage::MetricDefinition).to receive(:all).and_return([
instance_double(Gitlab::Usage::MetricDefinition, available?: true, attributes: { options: { events: %w[event1 event2] } }),
instance_double(Gitlab::Usage::MetricDefinition, available?: true, attributes: { options: { events: ['event3'] } }),
instance_double(Gitlab::Usage::MetricDefinition, available?: false, attributes: { options: { events: ['event4'] } })
])
event_names = described_class.known_events.pluck(:name) # events are returned as { name => event_name }
expect(event_names).to match_array(%w[event1 event2 event3]) # event4 is not available
end
end
describe '.track_event_in_context' do
before do
allow(described_class).to receive(:known_events).and_return(known_events)
end
context 'with valid context' do
where(:entity, :event_name, :context) do
entity1 | context_event | default_context
entity1 | context_event | ultimate_context
entity1 | context_event | gold_context
end
with_them do
it 'increments context event counter' do
expect(Gitlab::Redis::HLL).to receive(:add) do |kwargs|
expect(kwargs[:key]).to match(/^#{context}\_.*/)
end
described_class.track_event_in_context(event_name, values: entity, context: context)
end
end
end
context 'when sending empty context' do
it 'is not incrementing the counter' do
expect(Gitlab::Redis::HLL).not_to receive(:add)
described_class.track_event_in_context(context_event, values: entity1, context: '')
end
end
end
describe '.unique_events' do
context 'with events tracked in context' do
before do
allow(described_class).to receive(:known_events).and_return(known_events)
described_class.track_event_in_context(context_event, values: [entity1, entity3], context: default_context, time: 2.days.ago)
described_class.track_event_in_context(context_event, values: entity3, context: ultimate_context, time: 2.days.ago)
described_class.track_event_in_context(context_event, values: entity3, context: gold_context, time: 2.days.ago)
described_class.track_event_in_context(context_event, values: entity3, context: invalid_context, time: 2.days.ago)
described_class.track_event_in_context(context_event, values: [entity1, entity2], context: '', time: 2.weeks.ago)
end
subject(:unique_events) { described_class.unique_events(event_names: context_event, start_date: 4.weeks.ago, end_date: Date.current, context: context) }
context 'with correct arguments' do
where(:context, :value) do
ref(:default_context) | 2
ref(:ultimate_context) | 1
ref(:gold_context) | 1
'' | 0
end
with_them do
it { is_expected.to eq value }
end
end
context 'with invalid context' do
let(:context) { invalid_context }
let(:event_names) { context_event }
it 'raise error' do
expect { unique_events }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::InvalidContext)
end
end
end
end
describe '.track_event' do
before do
allow(described_class).to receive(:known_events).and_return(known_events)
end
context 'with settings usage ping disabled' do
before do
stub_application_setting(usage_ping_enabled: false)
end
context 'with license usage ping enabled' do
before do
# License.current.customer_service_enabled? == true
create_current_license(operational_metrics_enabled: true)
end
it 'tracks the event' do
expect(Gitlab::Redis::HLL).to receive(:add)
described_class.track_event(context_event, values: entity1, time: Date.current)
end
end
end
end
end
lib/gitlab/usage_data_counters/hll_redis_counter.rb
+ 10
− 39
View file @ 33e0e05d
......@@ -8,7 +8,6 @@ module HLLRedisCounter
EventError = Class.new(StandardError)
UnknownEvent = Class.new(EventError)
InvalidContext = Class.new(EventError)
# Track event on entity_id
# Increment a Redis HLL counter for unique event_name and entity_id
......@@ -29,29 +28,13 @@ def track_event(event_name, values:, time: Time.current)
track(values, event_name, time: time)
end
# Track unique events
#
# event_name - The event name.
# values - One or multiple values counted.
# context - Event context, plan level tracking.
# time - Time of the action, set to Time.current.
def track_event_in_context(event_name, values:, context:, time: Time.zone.now)
return if context.blank?
return unless context.in?(valid_context_list)
track(values, event_name, context: context, time: time)
end
# Count unique events for a given time range.
#
# event_names - The list of the events to count.
# start_date - The start date of the time range.
# end_date - The end date of the time range.
# context - Event context, plan level tracking. Available if set when tracking.
def unique_events(event_names:, start_date:, end_date:, context: '')
count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do
raise InvalidContext if context.present? && !context.in?(valid_context_list)
end
def unique_events(event_names:, start_date:, end_date:)
count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date)
end
def known_event?(event_name)
......@@ -68,7 +51,7 @@ def calculate_events_union(event_names:, start_date:, end_date:)
private
def track(values, event_name, context: '', time: Time.zone.now)
def track(values, event_name, time: Time.zone.now)
return unless ::ServicePing::ServicePingSettings.enabled?
event = event_for(event_name)
......@@ -77,7 +60,7 @@ def track(values, event_name, context: '', time: Time.zone.now)
return if event.blank?
return unless Feature.enabled?(:redis_hll_tracking, type: :ops)
Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH)
Gitlab::Redis::HLL.add(key: redis_key(event, time), value: values, expiry: KEY_EXPIRY_LENGTH)
rescue StandardError => e
# Ignore any exceptions unless is dev or test env
......@@ -85,27 +68,20 @@ def track(values, event_name, context: '', time: Time.zone.now)
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
end
# The array of valid context on which we allow tracking
def valid_context_list
Plan.all_plans
end
def count_unique_events(event_names:, start_date:, end_date:, context: '')
def count_unique_events(event_names:, start_date:, end_date:)
events = events_for(Array(event_names).map(&:to_s))
yield events if block_given?
keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context)
keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date)
return FALLBACK unless keys.any?
redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
end
def keys_for_aggregation(events:, start_date:, end_date:, context: '')
def keys_for_aggregation(events:, start_date:, end_date:)
end_date = end_date.end_of_week - 1.week
(start_date.to_date..end_date.to_date).map do |date|
events.map { |event| redis_key(event, date, context) }
events.map { |event| redis_key(event, date) }
end.flatten.uniq
end
......@@ -134,20 +110,15 @@ def events_for(event_names)
end
# Compose the key in order to store events daily or weekly
def redis_key(event, time, context = '')
def redis_key(event, time)
raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s)
key = "{#{REDIS_SLOT}}_#{event[:name]}"
year_week = time.strftime('%G-%V')
key = "#{key}-#{year_week}"
key = "#{context}_#{key}" if context.present?
key
"#{key}-#{year_week}"
end
end
end
end
end
Gitlab::UsageDataCounters::HLLRedisCounter.prepend_mod_with('Gitlab::UsageDataCounters::HLLRedisCounter')
spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
+ 1
− 85
View file @ 33e0e05d
......@@ -8,9 +8,6 @@
let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' }
let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' }
let(:default_context) { 'default' }
let(:invalid_context) { 'invalid' }
around do |example|
# We need to freeze to a reference time
# because visits are grouped by the week number in the year
......@@ -73,7 +70,6 @@
let(:no_slot) { 'no_slot' }
let(:different_aggregation) { 'different_aggregation' }
let(:custom_daily_event) { 'g_analytics_custom' }
let(:context_event) { 'context_event' }
let(:global_category) { 'global' }
let(:compliance_category) { 'compliance' }
......@@ -88,8 +84,7 @@
{ name: category_productivity_event },
{ name: compliance_slot_event },
{ name: no_slot },
{ name: different_aggregation },
{ name: context_event }
{ name: different_aggregation }
].map(&:with_indifferent_access)
end
......@@ -191,43 +186,6 @@
end
end
describe '.track_event_in_context' do
context 'with valid contex' do
it 'increments context event counter' do
expect(Gitlab::Redis::HLL).to receive(:add) do |kwargs|
expect(kwargs[:key]).to match(/^#{default_context}_.*/)
end
described_class.track_event_in_context(context_event, values: entity1, context: default_context)
end
it 'tracks events with multiple values' do
values = [entity1, entity2]
expect(Gitlab::Redis::HLL).to receive(:add).with(key: /g_analytics_contribution/,
value: values,
expiry: described_class::KEY_EXPIRY_LENGTH)
described_class.track_event_in_context(:g_analytics_contribution, values: values, context: default_context)
end
end
context 'with empty context' do
it 'does not increment a counter' do
expect(Gitlab::Redis::HLL).not_to receive(:add)
described_class.track_event_in_context(context_event, values: entity1, context: '')
end
end
context 'when sending invalid context' do
it 'does not increment a counter' do
expect(Gitlab::Redis::HLL).not_to receive(:add)
described_class.track_event_in_context(context_event, values: entity1, context: invalid_context)
end
end
end
describe '.unique_events' do
before do
# events in current week, should not be counted as week is not complete
......@@ -337,48 +295,6 @@
end
end
describe 'context level tracking' do
using RSpec::Parameterized::TableSyntax
let(:known_events) do
[
{ name: 'event_name_1' },
{ name: 'event_name_2' },
{ name: 'event_name_3' }
].map(&:with_indifferent_access)
end
before do
allow(described_class).to receive(:known_events).and_return(known_events)
allow(described_class).to receive(:categories).and_return(%w(category1 category2))
described_class.track_event_in_context('event_name_1', values: [entity1, entity3], context: default_context, time: 2.days.ago)
described_class.track_event_in_context('event_name_1', values: entity3, context: default_context, time: 2.days.ago)
described_class.track_event_in_context('event_name_1', values: entity3, context: invalid_context, time: 2.days.ago)
described_class.track_event_in_context('event_name_2', values: [entity1, entity2], context: '', time: 2.weeks.ago)
end
subject(:unique_events) { described_class.unique_events(event_names: event_names, start_date: 4.weeks.ago, end_date: Date.current, context: context) }
context 'with correct arguments' do
where(:event_names, :context, :value) do
['event_name_1'] | 'default' | 2
['event_name_1'] | '' | 0
['event_name_2'] | '' | 0
end
with_them do
it { is_expected.to eq value }
end
end
context 'with invalid context' do
it 'raise error' do
expect { described_class.unique_events(event_names: 'event_name_1', start_date: 4.weeks.ago, end_date: Date.current, context: invalid_context) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::InvalidContext)
end
end
end
describe '.calculate_events_union' do
let(:time_range) { { start_date: 7.days.ago, end_date: DateTime.current } }
let(:known_events) do
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment