diff --git a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb new file mode 100644 index 0000000000000000000000000000000000000000..0d4b31ecc9967ede1e09b91bd27673ee3da8ee60 --- /dev/null +++ b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event| + if event[:aggregation].to_sym == :daily + migrate_daily_aggregated(event) + else + migrate_weekly_aggregated(event) + end + end + end + + def down + # no-op + end + + private + + def migrate_daily_aggregated(event) + days_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH + start_date = Date.today - days_back - 1.day + end_date = Date.today + 1.day + + (start_date..end_date).each do |date| + rename_key(event, date) + end + end + + def migrate_weekly_aggregated(event) + weeks_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH + start_date = (Date.today - weeks_back).beginning_of_week - 1.day + end_date = Date.today.end_of_week + 1.day + + (start_date..end_date).step(7).each { |date| rename_key(event, date) } + end + + def rename_key(event, date) + old_key = old_redis_key(event, date) + new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, date) + + # cannot simply rename due to different slots + Gitlab::Redis::SharedState.with do |redis| + hll_blob = redis.get(old_key) + + break unless hll_blob + + temp_key = new_key + "_#{Time.current.to_i}" + ttl = redis.ttl(old_key) + ttl = ttl > 0 ? ttl : Gitlab::UsageDataCounters::HLLRedisCounter.send(:expiry, event) + + redis.multi do |multi| + multi.set(temp_key, hll_blob, ex: 1.day.to_i) + multi.pfmerge(new_key, new_key, temp_key) + multi.expire(new_key, ttl) + end + + redis.del(temp_key) + end + end + + def old_redis_key(event, time) + name_with_slot = if event[:redis_slot].present? + event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}") + else + "{#{event[:name]}}" + end + + Gitlab::UsageDataCounters::HLLRedisCounter.send(:apply_time_aggregation, name_with_slot, time, event) + end +end diff --git a/db/schema_migrations/20230328111013 b/db/schema_migrations/20230328111013 new file mode 100644 index 0000000000000000000000000000000000000000..18b085e1df1e2d4c0815c46d60c605025fd502ea --- /dev/null +++ b/db/schema_migrations/20230328111013 @@ -0,0 +1 @@ +6acd119e30d05fc794998cd4a80416cb91e11021659eb62ac93590175039a6cf \ No newline at end of file diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/analytics.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/analytics.yml index a68a5109f2eea3d3f5f7bd3c8fb6c7948163d7ca..93b78d6da562f2242d032a5394fa7714b6a5013f 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/analytics.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/analytics.yml @@ -1,28 +1,42 @@ - name: g_analytics_ci_cd_change_failure_rate + redis_slot: analytics aggregation: weekly - name: g_analytics_ci_cd_deployment_frequency + redis_slot: analytics aggregation: weekly - name: g_analytics_ci_cd_lead_time + redis_slot: analytics aggregation: weekly - name: g_analytics_ci_cd_release_statistics + redis_slot: analytics aggregation: weekly - name: g_analytics_ci_cd_time_to_restore_service + redis_slot: analytics aggregation: weekly - name: g_analytics_contribution + redis_slot: analytics aggregation: weekly - name: g_analytics_insights + redis_slot: analytics aggregation: weekly - name: p_analytics_insights + redis_slot: analytics aggregation: weekly - name: g_analytics_issues + redis_slot: analytics aggregation: weekly - name: p_analytics_issues + redis_slot: analytics aggregation: weekly - name: p_analytics_code_reviews + redis_slot: analytics aggregation: weekly - name: p_analytics_merge_request + redis_slot: analytics aggregation: weekly - name: g_analytics_productivity + redis_slot: analytics aggregation: weekly - name: g_analytics_valuestream + redis_slot: analytics aggregation: weekly diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/code_review_events.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/code_review_events.yml index a5553d6b928c5ef1445511cf706ecbe8f496d0ad..cbab50e1ae6834e555227fd63119a3036b19ef40 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,30 +1,44 @@ - name: i_code_review_mr_with_invalid_approvers + redis_slot: code_review aggregation: weekly ## Status Checks - name: i_code_review_merge_request_widget_status_checks_expand_failed + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_status_checks_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_status_checks_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_status_checks_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_status_checks_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_status_checks_view + redis_slot: code_review aggregation: weekly ## Metrics - name: i_code_review_merge_request_widget_license_compliance_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_metrics_expand_failed + redis_slot: code_review aggregation: weekly diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml index 779e4a0cb572931b09dc277c1baa54684897214f..ffb79929e9ab6a3e91cbd58d63ab584293926b8b 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml @@ -1,62 +1,89 @@ --- # Incident management - name: incident_management_incident_published + redis_slot: incident_management aggregation: weekly # Incident management on-call - name: i_incident_management_oncall_notification_sent + redis_slot: incident_management aggregation: weekly - name: i_testing_full_code_quality_report_total + redis_slot: testing aggregation: weekly - name: i_testing_group_code_coverage_project_click_total + redis_slot: testing aggregation: weekly - name: i_testing_group_code_coverage_visit_total + redis_slot: testing aggregation: weekly - name: i_testing_metrics_report_artifact_uploaders + redis_slot: testing aggregation: weekly - name: i_search_advanced + redis_slot: search aggregation: weekly - name: i_search_paid + redis_slot: search aggregation: weekly - name: a_compliance_audit_events_api + redis_slot: compliance aggregation: weekly - name: g_compliance_audit_events + redis_slot: compliance aggregation: weekly - name: g_compliance_dashboard + redis_slot: compliance aggregation: weekly - name: i_compliance_audit_events + redis_slot: compliance aggregation: weekly - name: i_compliance_credential_inventory + redis_slot: compliance aggregation: weekly - name: approval_project_rule_created + redis_slot: approval_project_rule aggregation: daily # Incident management linked resources - name: incident_management_issuable_resource_link_created + redis_slot: incident_management aggregation: weekly - name: incident_management_issuable_resource_link_deleted + redis_slot: incident_management aggregation: weekly - name: incident_management_issuable_resource_link_visited + redis_slot: incident_management aggregation: weekly # Project Management group - name: g_project_management_issue_added_to_epic + redis_slot: project_management aggregation: daily - name: g_project_management_issue_changed_epic + redis_slot: project_management aggregation: daily - name: g_project_management_issue_health_status_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_iteration_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_removed_from_epic + redis_slot: project_management aggregation: daily - name: g_project_management_issue_weight_changed + redis_slot: project_management aggregation: daily - name: i_ci_secrets_management_vault_build_created + redis_slot: ci_secrets_management aggregation: weekly # Container Security - Network Policies - name: clusters_using_network_policies_ui + redis_slot: network_policies aggregation: weekly # Geo group - name: g_geo_proxied_requests + redis_slot: geo aggregation: daily # Growth - name: users_clicking_registration_features_offer + redis_slot: users aggregation: weekly diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/ecosystem.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/ecosystem.yml index 3c115813c7854f988dfd74df4852f07ea6d26b32..fc22089c1b34275c30d615fc8e4bd2cbea8b3650 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -1,4 +1,6 @@ - name: i_ecosystem_jira_service_create_issue + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_jira_service_list_issues + redis_slot: ecosystem aggregation: weekly diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_board_events.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_board_events.yml index 8165308f787230e508eafcf775d45f886e7d2ac3..2df01dcf8e98225e4d1a7004a9893eb33181d6c0 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_board_events.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_board_events.yml @@ -4,10 +4,13 @@ # epic events to allow data aggregation. # More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 - name: g_project_management_users_creating_epic_boards + redis_slot: project_management aggregation: daily - name: g_project_management_users_viewing_epic_boards + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_epic_board_names + redis_slot: project_management aggregation: daily diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_events.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_events.yml index 51529f280a0eb2d60ec9079590173185ccc7e884..57a480f885817a128a9d1215ebeee00453e0b44d 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/epic_events.yml @@ -4,118 +4,153 @@ # epic events to allow data aggregation. # More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 - name: g_project_management_epic_created + redis_slot: project_management aggregation: daily # content change events - name: project_management_users_unchecking_epic_task + redis_slot: project_management aggregation: daily - name: project_management_users_checking_epic_task + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_epic_titles + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_epic_descriptions + redis_slot: project_management aggregation: daily # epic notes - name: g_project_management_users_creating_epic_notes + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_epic_notes + redis_slot: project_management aggregation: daily - name: g_project_management_users_destroying_epic_notes + redis_slot: project_management aggregation: daily # emoji - name: g_project_management_users_awarding_epic_emoji + redis_slot: project_management aggregation: daily - name: g_project_management_users_removing_epic_emoji + redis_slot: project_management aggregation: daily # start date events - name: g_project_management_users_setting_epic_start_date_as_fixed + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_fixed_epic_start_date + redis_slot: project_management aggregation: daily - name: g_project_management_users_setting_epic_start_date_as_inherited + redis_slot: project_management aggregation: daily # due date events - name: g_project_management_users_setting_epic_due_date_as_fixed + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_fixed_epic_due_date + redis_slot: project_management aggregation: daily - name: g_project_management_users_setting_epic_due_date_as_inherited + redis_slot: project_management aggregation: daily # relationships - name: g_project_management_epic_issue_added + redis_slot: project_management aggregation: daily - name: g_project_management_epic_issue_removed + redis_slot: project_management aggregation: daily - name: g_project_management_epic_issue_moved_from_project + redis_slot: project_management aggregation: daily - name: g_project_management_users_updating_epic_parent + redis_slot: project_management aggregation: daily - name: g_project_management_epic_closed + redis_slot: project_management aggregation: daily - name: g_project_management_epic_reopened + redis_slot: project_management aggregation: daily - name: 'g_project_management_issue_promoted_to_epic' + redis_slot: project_management aggregation: daily - name: g_project_management_users_setting_epic_confidential + redis_slot: project_management aggregation: daily - name: g_project_management_users_setting_epic_visible + redis_slot: project_management aggregation: daily - name: g_project_management_epic_users_changing_labels + redis_slot: project_management aggregation: daily - name: g_project_management_epic_destroyed + redis_slot: project_management aggregation: daily - name: g_project_management_epic_cross_referenced + redis_slot: project_management aggregation: daily - name: g_project_management_users_epic_issue_added_from_epic + redis_slot: project_management aggregation: daily - name: g_project_management_epic_related_added + redis_slot: project_management aggregation: daily - name: g_project_management_epic_related_removed + redis_slot: project_management aggregation: daily - name: g_project_management_epic_blocking_added + redis_slot: project_management aggregation: daily - name: g_project_management_epic_blocking_removed + redis_slot: project_management aggregation: daily - name: g_project_management_epic_blocked_added + redis_slot: project_management aggregation: daily - name: g_project_management_epic_blocked_removed + redis_slot: project_management aggregation: daily diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/integrations.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/integrations.yml index 4a83581e9f0d187bc99aedc65dc64c129b1398d6..f6367ae64369afadf367a30f24f2357c3defec71 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/integrations.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/integrations.yml @@ -1,18 +1,27 @@ - name: i_integrations_gitlab_for_slack_app_issue_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_push_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_deployment_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_wiki_page_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_merge_request_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_note_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_tag_push_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_confidential_note_notification + redis_slot: integrations aggregation: weekly - name: i_integrations_gitlab_for_slack_app_confidential_issue_notification + redis_slot: integrations aggregation: weekly diff --git a/ee/lib/ee/gitlab/usage_data_counters/known_events/quickactions.yml b/ee/lib/ee/gitlab/usage_data_counters/known_events/quickactions.yml index f809c6ca42d2cadc627bcd2480be2a6b1238a08c..912c91572297a0903c646d6ad012c77cb11a23b5 100644 --- a/ee/lib/ee/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/ee/lib/ee/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,30 +1,45 @@ - name: i_quickactions_child_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_clear_health_status + redis_slot: quickactions aggregation: weekly - name: i_quickactions_clear_weight + redis_slot: quickactions aggregation: weekly - name: i_quickactions_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_health_status + redis_slot: quickactions aggregation: weekly - name: i_quickactions_iteration + redis_slot: quickactions aggregation: weekly - name: i_quickactions_page + redis_slot: quickactions aggregation: weekly - name: i_quickactions_parent_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_promote + redis_slot: quickactions aggregation: weekly - name: i_quickactions_publish + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_child_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_iteration + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_parent_epic + redis_slot: quickactions aggregation: weekly - name: i_quickactions_weight + redis_slot: quickactions aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 0b30308b5525c0e834fc70d407c0deee0b3cbe37..1c390f2d7fd49dfe038144614dbd11d5d5548a9d 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -1,26 +1,39 @@ - name: users_viewing_analytics_group_devops_adoption + redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_adoption + redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_score + redis_slot: analytics aggregation: weekly - name: i_analytics_instance_statistics + redis_slot: analytics aggregation: weekly - name: p_analytics_pipelines + redis_slot: analytics aggregation: weekly - name: p_analytics_valuestream + redis_slot: analytics aggregation: weekly - name: p_analytics_repo + redis_slot: analytics aggregation: weekly - name: i_analytics_cohorts + redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_pipelines + redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_deployment_frequency + redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_lead_time + redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_time_to_restore_service + redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_change_failure_rate + redis_slot: analytics aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 82c023e6e38046b2b9bf73aedf17372da850931b..e717679e3dc7c95bbb43fd7f0d4f778f5214c1c3 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -4,304 +4,455 @@ # Do not edit it manually! --- - name: p_ci_templates_terraform_base_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_base + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_nodejs + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_openshift + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_auto_devops + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_bash + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_rust + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_elixir + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_clojure + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_crystal + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_getting_started + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_code_quality + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_load_performance_testing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_accessibility + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_failfast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_grails + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_runner_validation + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_scan + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secret_detection + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_license_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_api_scan + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secure_binaries + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dependency_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_discovery + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_fortify_fod_sast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_qualys_iac_security + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ios_fastlane + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_composer + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_c + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_python + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_fastlane + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_django + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_maven + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_liquibase + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_flutter + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_branch_pipelines + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_mergerequest_pipelines + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_laravel + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_kaniko + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_php + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_packer + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_themekit + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_katalon + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_mono + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_go + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_scala + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_latex + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_indeni_cloudrail + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_matlab + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_deploy_ecs + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_cf_provision_and_deploy_ec2 + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_deploy_ecs + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_gradle + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_chef + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dast_default_branch_deploy + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_load_performance_testing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_helm_2to3 + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_intelligence + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_quality + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ecs + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ec2 + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_test + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_cf_provision + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_latest + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_swift + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jekyll + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_harp + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_octopress + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_brunch + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_doxygen + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hyde + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_lektor + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jbake + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hexo + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_middleman + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hugo + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_pelican + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_nanoc + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_swaggerui + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jigsaw + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_metalsmith + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_gatsby + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_html + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dart + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_docker + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_julia + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_npm + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet_core + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_5_minute_production_app + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ruby + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_auto_devops + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_browser_performance_testing + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_build + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_intelligence + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_quality + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_container_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dependency_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ec2 + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ecs + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_helm_2to3 + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_license_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_sast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_secret_detection + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_test + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_container_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dependency_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_license_scanning + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_sast + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_secret_detection + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module_base + redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module + redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index 49757c6e672bcf0503b9f76378404767065497a2..6db10366b839d704c1308b112f60512048f020ce 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -1,4 +1,6 @@ - name: ci_users_executing_deployment_job + redis_slot: ci_users aggregation: weekly - name: ci_users_executing_verify_environment_job + redis_slot: ci_users aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index db0c0653f636b3dec3a9587b168ed2a8a332e91a..f64da801c39efcdee710eb9c3746022029bd8ae5 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,233 +1,345 @@ --- - name: i_code_review_create_note_in_ipynb_diff + redis_slot: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_commit + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_commit + redis_slot: code_review aggregation: weekly - name: i_code_review_mr_diffs + redis_slot: code_review aggregation: weekly - name: i_code_review_user_single_file_diffs + redis_slot: code_review aggregation: weekly - name: i_code_review_mr_single_file_diffs + redis_slot: code_review aggregation: weekly - name: i_code_review_user_toggled_task_item_status + redis_slot: code_review aggregation: weekly - name: i_code_review_create_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_close_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_reopen_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_approve_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_unapprove_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_resolve_thread + redis_slot: code_review aggregation: weekly - name: i_code_review_user_unresolve_thread + redis_slot: code_review aggregation: weekly - name: i_code_review_edit_mr_title + redis_slot: code_review aggregation: weekly - name: i_code_review_edit_mr_desc + redis_slot: code_review aggregation: weekly - name: i_code_review_user_merge_mr + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_edit_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_remove_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_review_note + redis_slot: code_review aggregation: weekly - name: i_code_review_user_publish_review + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_multiline_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_edit_multiline_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_remove_multiline_mr_comment + redis_slot: code_review aggregation: weekly - name: i_code_review_user_add_suggestion + redis_slot: code_review aggregation: weekly - name: i_code_review_user_apply_suggestion + redis_slot: code_review aggregation: weekly - name: i_code_review_user_assigned + redis_slot: code_review aggregation: weekly - name: i_code_review_user_marked_as_draft + redis_slot: code_review aggregation: weekly - name: i_code_review_user_unmarked_as_draft + redis_slot: code_review aggregation: weekly - name: i_code_review_user_review_requested + redis_slot: code_review aggregation: weekly - name: i_code_review_user_approval_rule_added + redis_slot: code_review aggregation: weekly - name: i_code_review_user_approval_rule_deleted + redis_slot: code_review aggregation: weekly - name: i_code_review_user_approval_rule_edited + redis_slot: code_review aggregation: weekly - name: i_code_review_user_vs_code_api_request + redis_slot: code_review aggregation: weekly - name: i_code_review_user_jetbrains_api_request + redis_slot: code_review aggregation: weekly - name: i_code_review_user_gitlab_cli_api_request + redis_slot: code_review aggregation: weekly - name: i_code_review_user_create_mr_from_issue + redis_slot: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_locked + redis_slot: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_unlocked + redis_slot: code_review aggregation: weekly - name: i_code_review_user_time_estimate_changed + redis_slot: code_review aggregation: weekly - name: i_code_review_user_time_spent_changed + redis_slot: code_review aggregation: weekly - name: i_code_review_user_assignees_changed + redis_slot: code_review aggregation: weekly - name: i_code_review_user_reviewers_changed + redis_slot: code_review aggregation: weekly - name: i_code_review_user_milestone_changed + redis_slot: code_review aggregation: weekly - name: i_code_review_user_labels_changed + redis_slot: code_review aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting + redis_slot: code_review aggregation: weekly - name: i_code_review_click_single_file_mode_setting + redis_slot: code_review aggregation: weekly - name: i_code_review_click_file_browser_setting + redis_slot: code_review aggregation: weekly - name: i_code_review_click_whitespace_setting + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_view_inline + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_view_parallel + redis_slot: code_review aggregation: weekly - name: i_code_review_file_browser_tree_view + redis_slot: code_review aggregation: weekly - name: i_code_review_file_browser_list_view + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_show_whitespace + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_hide_whitespace + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_single_file + redis_slot: code_review aggregation: weekly - name: i_code_review_diff_multiple_files + redis_slot: code_review aggregation: weekly - name: i_code_review_user_load_conflict_ui + redis_slot: code_review aggregation: weekly - name: i_code_review_user_resolve_conflict + redis_slot: code_review aggregation: weekly - name: i_code_review_user_searches_diff + redis_slot: code_review aggregation: weekly - name: i_code_review_total_suggestions_applied + redis_slot: code_review aggregation: weekly - name: i_code_review_total_suggestions_added + redis_slot: code_review aggregation: weekly - name: i_code_review_user_resolve_thread_in_issue + redis_slot: code_review aggregation: weekly - name: i_code_review_widget_nothing_merge_click_new_file + redis_slot: code_review aggregation: weekly - name: i_code_review_post_merge_delete_branch + redis_slot: code_review aggregation: weekly - name: i_code_review_post_merge_click_revert + redis_slot: code_review aggregation: weekly - name: i_code_review_post_merge_click_cherry_pick + redis_slot: code_review aggregation: weekly - name: i_code_review_post_merge_submit_revert_modal + redis_slot: code_review aggregation: weekly - name: i_code_review_post_merge_submit_cherry_pick_modal + redis_slot: code_review aggregation: weekly # MR Widget Extensions ## Test Summary - name: i_code_review_merge_request_widget_test_summary_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_failed + redis_slot: code_review aggregation: weekly ## Accessibility - name: i_code_review_merge_request_widget_accessibility_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_failed + redis_slot: code_review aggregation: weekly ## Code Quality - name: i_code_review_merge_request_widget_code_quality_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_failed + redis_slot: code_review aggregation: weekly ## Terraform - name: i_code_review_merge_request_widget_terraform_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_failed + redis_slot: code_review aggregation: weekly - name: i_code_review_submit_review_approve + redis_slot: code_review aggregation: weekly - name: i_code_review_submit_review_comment + redis_slot: code_review aggregation: weekly ## License Compliance - name: i_code_review_merge_request_widget_license_compliance_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_failed + redis_slot: code_review aggregation: weekly ## Security Reports - name: i_code_review_merge_request_widget_security_reports_view + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_full_report_clicked + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_success + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_warning + redis_slot: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_failed + redis_slot: code_review aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index d3520961665bfb783376e0b44625215786c9e812..3314997f873d26bf8c478ebd228f1a0e6961b2fd 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -1,14 +1,19 @@ --- # Compliance category - name: g_edit_by_web_ide + redis_slot: edit aggregation: daily - name: g_edit_by_sfe + redis_slot: edit aggregation: daily - name: g_edit_by_snippet_ide + redis_slot: edit aggregation: daily - name: g_edit_by_live_preview + redis_slot: edit aggregation: daily - name: i_search_total + redis_slot: search aggregation: weekly - name: wiki_action aggregation: daily @@ -21,145 +26,209 @@ - name: merge_request_action aggregation: daily - name: i_source_code_code_intelligence + redis_slot: source_code aggregation: daily # Incident management - name: incident_management_alert_status_changed + redis_slot: incident_management aggregation: weekly - name: incident_management_alert_assigned + redis_slot: incident_management aggregation: weekly - name: incident_management_alert_todo + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_created + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_reopened + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_closed + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_assigned + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_todo + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_comment + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_zoom_meeting + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_relate + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_unrelate + redis_slot: incident_management aggregation: weekly - name: incident_management_incident_change_confidential + redis_slot: incident_management aggregation: weekly # Incident management timeline events - name: incident_management_timeline_event_created + redis_slot: incident_management aggregation: weekly - name: incident_management_timeline_event_edited + redis_slot: incident_management aggregation: weekly - name: incident_management_timeline_event_deleted + redis_slot: incident_management aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident + redis_slot: incident_management aggregation: weekly # Testing category - name: i_testing_test_case_parsed + redis_slot: testing aggregation: weekly - name: i_testing_test_report_uploaded + redis_slot: testing aggregation: weekly - name: i_testing_coverage_report_uploaded + redis_slot: testing aggregation: weekly # Project Management group - name: g_project_management_issue_title_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_description_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_assignee_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_confidential + redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_visible + redis_slot: project_management aggregation: daily - name: g_project_management_issue_created + redis_slot: project_management aggregation: daily - name: g_project_management_issue_closed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_reopened + redis_slot: project_management aggregation: daily - name: g_project_management_issue_label_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_milestone_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_cross_referenced + redis_slot: project_management aggregation: daily - name: g_project_management_issue_moved + redis_slot: project_management aggregation: daily - name: g_project_management_issue_related + redis_slot: project_management aggregation: daily - name: g_project_management_issue_unrelated + redis_slot: project_management aggregation: daily - name: g_project_management_issue_marked_as_duplicate + redis_slot: project_management aggregation: daily - name: g_project_management_issue_locked + redis_slot: project_management aggregation: daily - name: g_project_management_issue_unlocked + redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_added + redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_modified + redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_removed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_due_date_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_design_comments_removed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_estimate_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_spent_changed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_added + redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_edited + redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_removed + redis_slot: project_management aggregation: daily - name: g_project_management_issue_cloned + redis_slot: project_management aggregation: daily # Runner group - name: g_runner_fleet_read_jobs_statistics + redis_slot: runner aggregation: weekly # Secrets Management - name: i_snippets_show + redis_slot: snippets aggregation: weekly # Terraform - name: p_terraform_state_api_unique_users + redis_slot: terraform aggregation: weekly # Pipeline Authoring group - name: o_pipeline_authoring_unique_users_committing_ciconfigfile + redis_slot: pipeline_authoring aggregation: weekly - name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile + redis_slot: pipeline_authoring aggregation: weekly - name: i_ci_secrets_management_id_tokens_build_created + redis_slot: ci_secrets_management aggregation: weekly # Merge request widgets - name: users_expanding_secure_security_report + redis_slot: secure aggregation: weekly - name: users_expanding_testing_code_quality_report + redis_slot: testing aggregation: weekly - name: users_expanding_testing_accessibility_report + redis_slot: testing aggregation: weekly - name: users_expanding_testing_license_compliance_report + redis_slot: testing aggregation: weekly - name: users_visiting_testing_license_compliance_full_report + redis_slot: testing aggregation: weekly - name: users_visiting_testing_manage_license_compliance + redis_slot: testing aggregation: weekly - name: users_clicking_license_testing_visiting_external_website + redis_slot: testing aggregation: weekly # Geo group - name: g_geo_proxied_requests + redis_slot: geo aggregation: daily # Manage - name: unique_active_user aggregation: weekly # Environments page - name: users_visiting_environments_pages + redis_slot: users aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml index aa0f9965fa721b4c9c0f0e11c41d5fb5f53cf443..ac40079a6dc200c3f27afd1c7220b3466d9cfe09 100644 --- a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml @@ -1,11 +1,16 @@ --- - name: i_container_registry_push_tag_user aggregation: weekly + redis_slot: container_registry - name: i_container_registry_delete_tag_user aggregation: weekly + redis_slot: container_registry - name: i_container_registry_push_repository_user aggregation: weekly + redis_slot: container_registry - name: i_container_registry_delete_repository_user aggregation: weekly + redis_slot: container_registry - name: i_container_registry_create_repository_user aggregation: weekly + redis_slot: container_registry diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 6e4a893d19aecc9ed6cf216ef3c3287072a12740..03bbba663c52c1477f03a3698ada6f8925f9b57a 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -1,24 +1,35 @@ --- # Ecosystem category - name: i_ecosystem_jira_service_close_issue + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_jira_service_cross_reference + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_issue_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_push_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_deployment_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_wiki_page_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_merge_request_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_note_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_tag_push_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_note_notification + redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_issue_notification + redis_slot: ecosystem aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml index ebfd1b274f9d4de48bb9870e7423b5f9966c7f04..efed16c11f81479caad0ac58735b1982eef1ffb0 100644 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -1,5 +1,7 @@ --- - name: error_tracking_view_details + redis_slot: error_tracking aggregation: weekly - name: error_tracking_view_list + redis_slot: error_tracking aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml index 3346c0556d64e17fc16332f9e3b637913ccb6e6a..a6c90a6c762ba223a36d06c8c997f6e247286aff 100644 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -1,10 +1,13 @@ --- # Importer events - name: github_import_project_start + redis_slot: import aggregation: weekly - name: github_import_project_success + redis_slot: import aggregation: weekly - name: github_import_project_failure + redis_slot: import aggregation: weekly - name: github_import_project_cancelled redis_slot: import @@ -12,3 +15,4 @@ - name: github_import_project_partially_completed redis_slot: import aggregation: weekly + diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index b3d1c51c0e7424cad1b0ae3de0ee9fdd87f74c96..9703c022ef5dec83192d8b8b6723d775b1302044 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -1,2 +1,3 @@ - name: agent_users_using_ci_tunnel + redis_slot: agent aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index 47cc7f98838a161579dc1d7553a101d2e3760eff..d9797635240630abb86c35ef0b260250aeff7e06 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -1,45 +1,67 @@ --- - name: i_package_composer_deploy_token aggregation: weekly + redis_slot: package - name: i_package_composer_user aggregation: weekly + redis_slot: package - name: i_package_conan_deploy_token aggregation: weekly + redis_slot: package - name: i_package_conan_user aggregation: weekly + redis_slot: package - name: i_package_generic_deploy_token aggregation: weekly + redis_slot: package - name: i_package_generic_user aggregation: weekly + redis_slot: package - name: i_package_helm_deploy_token aggregation: weekly + redis_slot: package - name: i_package_helm_user aggregation: weekly + redis_slot: package - name: i_package_maven_deploy_token aggregation: weekly + redis_slot: package - name: i_package_maven_user aggregation: weekly + redis_slot: package - name: i_package_npm_deploy_token aggregation: weekly + redis_slot: package - name: i_package_npm_user aggregation: weekly + redis_slot: package - name: i_package_nuget_deploy_token aggregation: weekly + redis_slot: package - name: i_package_nuget_user aggregation: weekly + redis_slot: package - name: i_package_pypi_deploy_token aggregation: weekly + redis_slot: package - name: i_package_pypi_user aggregation: weekly + redis_slot: package - name: i_package_rubygems_deploy_token aggregation: weekly + redis_slot: package - name: i_package_rubygems_user aggregation: weekly + redis_slot: package - name: i_package_terraform_module_deploy_token aggregation: weekly + redis_slot: package - name: i_package_terraform_module_user aggregation: weekly + redis_slot: package - name: i_package_rpm_user aggregation: weekly + redis_slot: package - name: i_package_rpm_deploy_token aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 7006173cc591a93e5f5b96d91fbd78dfd6f61caa..306ed79ea239080dd4c446ec31f570c9dbe83bd7 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,127 +1,190 @@ --- - name: i_quickactions_assign_multiple + redis_slot: quickactions aggregation: weekly - name: i_quickactions_approve + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unapprove + redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_single + redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_self + redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_reviewer + redis_slot: quickactions aggregation: weekly - name: i_quickactions_award + redis_slot: quickactions aggregation: weekly - name: i_quickactions_board_move + redis_slot: quickactions aggregation: weekly - name: i_quickactions_clone + redis_slot: quickactions aggregation: weekly - name: i_quickactions_close + redis_slot: quickactions aggregation: weekly - name: i_quickactions_confidential + redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_merge_request + redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_issue + redis_slot: quickactions aggregation: weekly - name: i_quickactions_create_merge_request + redis_slot: quickactions aggregation: weekly - name: i_quickactions_done + redis_slot: quickactions aggregation: weekly - name: i_quickactions_draft + redis_slot: quickactions aggregation: weekly - name: i_quickactions_due + redis_slot: quickactions aggregation: weekly - name: i_quickactions_duplicate + redis_slot: quickactions aggregation: weekly - name: i_quickactions_estimate + redis_slot: quickactions aggregation: weekly - name: i_quickactions_label + redis_slot: quickactions aggregation: weekly - name: i_quickactions_lock + redis_slot: quickactions aggregation: weekly - name: i_quickactions_merge + redis_slot: quickactions aggregation: weekly - name: i_quickactions_milestone + redis_slot: quickactions aggregation: weekly - name: i_quickactions_move + redis_slot: quickactions aggregation: weekly - name: i_quickactions_promote_to_incident + redis_slot: quickactions aggregation: weekly - name: i_quickactions_timeline + redis_slot: quickactions aggregation: weekly - name: i_quickactions_ready + redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign + redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign_reviewer + redis_slot: quickactions aggregation: weekly - name: i_quickactions_rebase + redis_slot: quickactions aggregation: weekly - name: i_quickactions_relabel + redis_slot: quickactions aggregation: weekly - name: i_quickactions_relate + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_due_date + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_estimate + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_milestone + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_time_spent + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_zoom + redis_slot: quickactions aggregation: weekly - name: i_quickactions_reopen + redis_slot: quickactions aggregation: weekly - name: i_quickactions_severity + redis_slot: quickactions aggregation: weekly - name: i_quickactions_shrug + redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_subtract + redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_add + redis_slot: quickactions aggregation: weekly - name: i_quickactions_submit_review + redis_slot: quickactions aggregation: weekly - name: i_quickactions_subscribe + redis_slot: quickactions aggregation: weekly - name: i_quickactions_tableflip + redis_slot: quickactions aggregation: weekly - name: i_quickactions_tag + redis_slot: quickactions aggregation: weekly - name: i_quickactions_target_branch + redis_slot: quickactions aggregation: weekly - name: i_quickactions_title + redis_slot: quickactions aggregation: weekly - name: i_quickactions_todo + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_specific + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_all + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_reviewer + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_specific + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_all + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlock + redis_slot: quickactions aggregation: weekly - name: i_quickactions_unsubscribe + redis_slot: quickactions aggregation: weekly - name: i_quickactions_wip + redis_slot: quickactions aggregation: weekly - name: i_quickactions_zoom + redis_slot: quickactions aggregation: weekly - name: i_quickactions_link + redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_single + redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_multiple + redis_slot: quickactions aggregation: weekly - name: i_quickactions_add_contacts + redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_contacts + redis_slot: quickactions aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index a6e5b9e1af52bc017508f3bde1048d57c564a6f9..1f0cc0c8a2e596bc9af07d8cea3cb43b40797152 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -1,21 +1,28 @@ --- - name: users_updating_work_item_title + redis_slot: users aggregation: weekly - name: users_creating_work_items + redis_slot: users aggregation: weekly - name: users_updating_work_item_dates + redis_slot: users aggregation: weekly - name: users_updating_work_item_labels + redis_slot: users aggregation: weekly - name: users_updating_work_item_milestone + redis_slot: users aggregation: weekly - name: users_updating_work_item_iteration # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. + redis_slot: users aggregation: weekly - name: users_updating_weight_estimate # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. + redis_slot: users aggregation: weekly diff --git a/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..4c6d4907c29ab8046471fe6e978876e4f6c409db --- /dev/null +++ b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe ReMigrateRedisSlotKeys, :migration, feature_category: :service_ping do + let(:date) { Date.yesterday.strftime('%G-%j') } + let(:week) { Date.yesterday.strftime('%G-%V') } + let(:known_events) do + [ + { + redis_slot: 'analytics', + aggregation: 'daily', + name: 'users_viewing_analytics_group_devops_adoption' + }, { + aggregation: 'weekly', + name: 'wiki_action' + }, { + aggregation: 'weekly', + name: 'non_existing_event' + }, { + aggregation: 'weekly', + name: 'event_without_expiry' + } + ] + end + + describe "#up" do + it 'rename keys', :aggregate_failures do + allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events) + .and_return(known_events) + + expiry_daily = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH + expiry_weekly = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH + + default_slot = Gitlab::UsageDataCounters::HLLRedisCounter::REDIS_SLOT + + old_slot_a = "#{date}-users_viewing_{analytics}_group_devops_adoption" + old_slot_b = "{wiki_action}-#{week}" + old_slot_without_expiry = "{event_without_expiry}-#{week}" + + new_slot_a = "#{date}-{#{default_slot}}_users_viewing_analytics_group_devops_adoption" + new_slot_b = "{#{default_slot}}_wiki_action-#{week}" + new_slot_without_expiry = "{#{default_slot}}_event_without_expiry-#{week}" + + Gitlab::Redis::HLL.add(key: old_slot_a, value: 1, expiry: expiry_daily) + Gitlab::Redis::HLL.add(key: old_slot_b, value: 1, expiry: expiry_weekly) + Gitlab::Redis::HLL.add(key: old_slot_a, value: 2, expiry: expiry_daily) + Gitlab::Redis::HLL.add(key: old_slot_b, value: 2, expiry: expiry_weekly) + Gitlab::Redis::HLL.add(key: old_slot_b, value: 2, expiry: expiry_weekly) + Gitlab::Redis::SharedState.with { |redis| redis.pfadd(old_slot_without_expiry, 1) } + + # check that we merge values during migration + # i.e. we dont drop keys created after code deploy but before the migration + Gitlab::Redis::HLL.add(key: new_slot_a, value: 3, expiry: expiry_daily) + Gitlab::Redis::HLL.add(key: new_slot_b, value: 3, expiry: expiry_weekly) + Gitlab::Redis::HLL.add(key: new_slot_without_expiry, value: 2, expiry: expiry_weekly) + + migrate! + + expect(Gitlab::Redis::HLL.count(keys: new_slot_a)).to eq(3) + expect(Gitlab::Redis::HLL.count(keys: new_slot_b)).to eq(3) + expect(Gitlab::Redis::HLL.count(keys: new_slot_without_expiry)).to eq(2) + expect(with_redis { |r| r.ttl(new_slot_a) }).to be_within(600).of(expiry_daily) + expect(with_redis { |r| r.ttl(new_slot_b) }).to be_within(600).of(expiry_weekly) + expect(with_redis { |r| r.ttl(new_slot_without_expiry) }).to be_within(600).of(expiry_weekly) + end + + it 'runs without errors' do + expect { migrate! }.not_to raise_error + end + end + + def with_redis(&block) + Gitlab::Redis::SharedState.with(&block) + end +end