Bypass category question tracking in on-premises instances

What does this MR do and why?

This MR bypasses category question tracking in on-premises instances. The snowplow tracking is only available in SaaS instance.

MR acceptance checklist

Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.

Screenshots or screen recordings

Sidekiq execution log when a user asks a question in Duo Chat. You can see that two Llm::CompletionWorker executed in a single request:

{"severity":"INFO","time":"2023-12-28T05:37:17.892Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"004763f794de98b72cb0be94","created_at":"2023-12-28T05:37:17.859Z","correlation_id":"01HJQETMAGNF82ZZGQEBY0WQNB","meta.caller_id":"graphql:unknown","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:84f35fc252333b2abc0601bdc48936004c9050182ba7ad87fe3706ff2574a208","size_limiter":"validated","enqueued_at":"2023-12-28T05:37:17.892Z","job_size_bytes":584,"pid":72583,"message":"Llm::CompletionWorker JID-004763f794de98b72cb0be94: start","job_status":"start","scheduling_latency_s":0.000734}
{"severity":"WARN","time":"2023-12-28T05:37:22.249Z","message":"Job arguments to Llm::CompletionWorker do not serialize to JSON safely. This will raise an error in\nSidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today\nby calling `Sidekiq.strict_args!` during Sidekiq initialization.\n"}
{"severity":"INFO","time":"2023-12-28T05:37:22.250Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"d0a01d56ab7f278e491cd38b","created_at":"2023-12-28T05:37:22.248Z","correlation_id":"01HJQETMAGNF82ZZGQEBY0WQNB","meta.caller_id":"Llm::CompletionWorker","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:26d334e1867384548d43751f6fc76c322c4c83170d1b8cee702e7df228668305","size_limiter":"validated","enqueued_at":"2023-12-28T05:37:22.249Z","job_size_bytes":428,"pid":72583,"message":"Llm::CompletionWorker JID-d0a01d56ab7f278e491cd38b: start","job_status":"start","scheduling_latency_s":0.000753}
{"severity":"INFO","time":"2023-12-28T05:37:22.252Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"004763f794de98b72cb0be94","created_at":"2023-12-28T05:37:17.859Z","correlation_id":"01HJQETMAGNF82ZZGQEBY0WQNB","meta.caller_id":"graphql:unknown","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:84f35fc252333b2abc0601bdc48936004c9050182ba7ad87fe3706ff2574a208","size_limiter":"validated","enqueued_at":"2023-12-28T05:37:17.892Z","job_size_bytes":584,"pid":72583,"message":"Llm::CompletionWorker JID-004763f794de98b72cb0be94: done: 4.359568 sec","job_status":"done","scheduling_latency_s":0.000734,"redis_calls":21,"redis_duration_s":0.004784,"redis_read_bytes":7837,"redis_write_bytes":4183,"redis_cache_calls":14,"redis_cache_duration_s":0.00334,"redis_cache_read_bytes":7829,"redis_cache_write_bytes":1142,"redis_queues_calls":5,"redis_queues_duration_s":0.001018,"redis_queues_read_bytes":6,"redis_queues_write_bytes":1541,"redis_action_cable_calls":2,"redis_action_cable_duration_s":0.000426,"redis_action_cable_read_bytes":2,"redis_action_cable_write_bytes":1500,"db_count":4,"db_write_count":0,"db_cached_count":1,"db_replica_count":0,"db_primary_count":4,"db_main_count":4,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":1,"db_main_cached_count":1,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.01,"db_main_duration_s":0.01,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"external_http_count":1,"external_http_duration_s":4.324931283000069,"cpu_s":0.031617,"worker_id":"sidekiq_0","rate_limiting_gates":[],"duration_s":4.359568,"completed_at":"2023-12-28T05:37:22.252Z","load_balancing_strategy":"primary_no_wal","db_duration_s":0.004305,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
{"severity":"INFO","time":"2023-12-28T05:37:22.270Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"d0a01d56ab7f278e491cd38b","created_at":"2023-12-28T05:37:22.248Z","correlation_id":"01HJQETMAGNF82ZZGQEBY0WQNB","meta.caller_id":"Llm::CompletionWorker","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:26d334e1867384548d43751f6fc76c322c4c83170d1b8cee702e7df228668305","size_limiter":"validated","enqueued_at":"2023-12-28T05:37:22.249Z","job_size_bytes":428,"pid":72583,"message":"Llm::CompletionWorker JID-d0a01d56ab7f278e491cd38b: done: 0.020565 sec","job_status":"done","scheduling_latency_s":0.000753,"redis_calls":3,"redis_duration_s":0.009467999999999999,"redis_read_bytes":5932,"redis_write_bytes":189,"redis_cache_calls":1,"redis_cache_duration_s":0.009302,"redis_cache_read_bytes":5930,"redis_cache_write_bytes":17,"redis_queues_calls":2,"redis_queues_duration_s":0.000166,"redis_queues_read_bytes":2,"redis_queues_write_bytes":172,"db_count":2,"db_write_count":0,"db_cached_count":1,"db_replica_count":0,"db_primary_count":2,"db_main_count":2,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":1,"db_main_cached_count":1,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.001,"db_main_duration_s":0.001,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.016135,"worker_id":"sidekiq_0","rate_limiting_gates":[],"duration_s":0.020565,"completed_at":"2023-12-28T05:37:22.270Z","load_balancing_strategy":"primary_no_wal","db_duration_s":0.000444,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}

The one called for categorize_question failed with the following error in on-premises instances duo to ai client incompatibility:

{"severity":"ERROR","time":"2023-12-28T05:37:22.269Z","correlation_id":"01HJQETMAGNF82ZZGQEBY0WQNB","meta.caller_id":"Llm::CompletionWorker","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","message":"Error","class":"Gitlab::Llm::Anthropic::Completions::CategorizeQuestion","error":"JSON has an invalid format."}

With this MR, the second Llm::CompletionWorker invocation for categorize_question is bypassed in on-premises instances:

{"severity":"INFO","time":"2023-12-28T05:51:08.389Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"f28131f95b9e523f64120405","created_at":"2023-12-28T05:51:08.362Z","correlation_id":"01HJQFKZCM6W24NP425CN63W8D","meta.caller_id":"graphql:unknown","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:e69efaa4f5a10b939f68e6a5de4879b3eb3f9a466904984a7d2c485a1d4a12ef","size_limiter":"validated","enqueued_at":"2023-12-28T05:51:08.383Z","job_size_bytes":584,"pid":81936,"message":"Llm::CompletionWorker JID-f28131f95b9e523f64120405: start","job_status":"start","scheduling_latency_s":0.006155}
{"severity":"INFO","time":"2023-12-28T05:51:15.193Z","retry":25,"queue":"default","backtrace":true,"version":0,"args":["[FILTERED]","[FILTERED]"],"class":"Llm::CompletionWorker","jid":"f28131f95b9e523f64120405","created_at":"2023-12-28T05:51:08.362Z","correlation_id":"01HJQFKZCM6W24NP425CN63W8D","meta.caller_id":"graphql:unknown","meta.remote_ip":"172.16.123.2","meta.feature_category":"ai_abstraction_layer","meta.user":"root","meta.user_id":1,"meta.client_id":"user/1","meta.root_caller_id":"graphql:unknown","worker_data_consistency":"delayed","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:e69efaa4f5a10b939f68e6a5de4879b3eb3f9a466904984a7d2c485a1d4a12ef","size_limiter":"validated","enqueued_at":"2023-12-28T05:51:08.383Z","job_size_bytes":584,"pid":81936,"message":"Llm::CompletionWorker JID-f28131f95b9e523f64120405: done: 6.803578 sec","job_status":"done","scheduling_latency_s":0.006155,"redis_calls":18,"redis_duration_s":0.009604999999999999,"redis_read_bytes":7924,"redis_write_bytes":2687,"redis_cache_calls":13,"redis_cache_duration_s":0.005467,"redis_cache_read_bytes":7918,"redis_cache_write_bytes":981,"redis_queues_calls":2,"redis_queues_duration_s":0.000466,"redis_queues_read_bytes":2,"redis_queues_write_bytes":172,"redis_action_cable_calls":3,"redis_action_cable_duration_s":0.003672,"redis_action_cable_read_bytes":4,"redis_action_cable_write_bytes":1534,"db_count":4,"db_write_count":0,"db_cached_count":1,"db_replica_count":0,"db_primary_count":4,"db_main_count":4,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":1,"db_main_cached_count":1,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.01,"db_main_duration_s":0.01,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"external_http_count":1,"external_http_duration_s":5.9640778670000145,"external_http_slow_requests":[{"method":"POST","host":"127.0.0.1","port":5052,"path":"/v1/chat/agent","duration_s":5.964}],"cpu_s":0.799309,"worker_id":"sidekiq_0","rate_limiting_gates":[],"duration_s":6.803578,"completed_at":"2023-12-28T05:51:15.193Z","load_balancing_strategy":"primary_no_wal","db_duration_s":0.022437,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}

How to set up and validate locally

See !138274 (merged)