Do not index associated issues that are epic work item type

What does this MR do and why?

Corrective action from gitlab-com/gl-infra/production#18330 (closed)

There are a few cases where associated records are queued for indexing using ElasticAssociationIndexerWorker. That completely bypasses the checks for issues that are work item type epic which cannot be indexed into the issues index (due to not having a project_id field).

This MR prevents associations from indexing by:

  • adding a searchable scope to issue model
  • apply searchable scope to Issue class records before queueing for indexing

MR acceptance checklist

Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.

Screenshots or screen recordings

N/A

How to set up and validate locally

  1. enable gdk for elasticsearch and index everything
  2. create issues from a "non-admin" user that are epic work item type issues
  3. ban the non-admin user
  4. monitor the logs in sidekiq.log and elasticsearch.log to ensure the record is not queued

on master, you would see the undefined method for nil class error but on this branch you should not

error from `master` branch
2024-07-25_14:54:13.56063 rails-background-jobs            : {"severity":"WARN","time":"2024-07-25T14:54:13.560Z","retry":0,"queue":"default","backtrace":true,"version":0,"store":null,"queue_namespace":"cronjob","args":["1"],"class":"ElasticIndexBulkCronWorker","jid":"58d47dafe50ed286bee8dcc6","created_at":"2024-07-25T14:54:13.308Z","trace_propagation_headers":{"sentry-trace":"3778606643654bba839976194217ac73-0ab533a1498a4603","baggage":"sentry-trace_id=3778606643654bba839976194217ac73,sentry-environment=,sentry-release=2d9968d18a9"},"meta.caller_id":"Cronjob","correlation_id":"c7c05be76d30a73e5cec2cac3e0cb2c5","meta.root_caller_id":"Cronjob","meta.feature_category":"global_search","worker_data_consistency":"sticky","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:e299b1e1b7aa4f7fa219749dc82aa084982849f63721e1d1c8ae48fe38f6530b","size_limiter":"validated","enqueued_at":"2024-07-25T14:54:13.311Z","job_size_bytes":3,"pid":64347,"message":"ElasticIndexBulkCronWorker JID-58d47dafe50ed286bee8dcc6: fail: 0.135342 sec","job_status":"fail","scheduling_latency_s":0.113117,"db_count":5,"db_write_count":0,"db_cached_count":0,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":5,"db_main_count":5,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_max_duration_s":0.0,"db_primary_txn_max_duration_s":0.0,"db_main_txn_max_duration_s":0.0,"db_ci_txn_max_duration_s":0.0,"db_main_replica_txn_max_duration_s":0.0,"db_ci_replica_txn_max_duration_s":0.0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.028,"db_main_duration_s":0.028,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.01412,"worker_id":"sidekiq_0","rate_limiting_gates":[],"exclusive_lock_requested_count":1,"exclusive_lock_wait_duration_s":0.001162,"exclusive_lock_hold_duration_s":0.021507,"duration_s":0.135342,"completed_at":"2024-07-25T14:54:13.560Z","load_balancing_strategy":"primary_no_wal","exception.class":"NoMethodError","exception.message":"undefined method `visibility_level' for nil:NilClass","exception.backtrace":["ee/lib/elastic/latest/issue_instance_proxy.rb:27:in `as_indexed_json'","ee/lib/elastic/multi_version_util.rb:76:in `as_indexed_json'","ee/lib/gitlab/elastic/document_reference.rb:125:in `as_indexed_json'","ee/lib/gitlab/elastic/bulk_indexer.rb:178:in `upsert_operation'","ee/lib/gitlab/elastic/bulk_indexer.rb:78:in `upsert'","ee/lib/gitlab/elastic/bulk_indexer.rb:46:in `process'","ee/app/services/elastic/process_bookkeeping_service.rb:267:in `submit_document'","ee/app/services/elastic/process_bookkeeping_service.rb:187:in `block in execute_with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:186:in `each'","ee/app/services/elastic/process_bookkeeping_service.rb:186:in `execute_with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:148:in `block in execute'","lib/gitlab/redis/wrapper.rb:25:in `block in with'","lib/gitlab/redis/wrapper.rb:25:in `with'","ee/app/services/elastic/process_bookkeeping_service.rb:106:in `with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:148:in `execute'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:50:in `block in process_shard'","lib/gitlab/exclusive_lease_helpers.rb:43:in `block in in_lock'","lib/gitlab/exclusive_lease_helpers.rb:53:in `with_instrumentation'","lib/gitlab/exclusive_lease_helpers.rb:42:in `in_lock'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:49:in `process_shard'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:34:in `perform'","ee/app/workers/concerns/geo/skip_secondary.rb:14:in `perform'","ee/lib/gitlab/sidekiq_middleware/set_session/server.rb:21:in `call'","lib/gitlab/sidekiq_middleware/skip_jobs.rb:51:in `call'","lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'","lib/gitlab/sidekiq_middleware/concurrency_limit/middleware.rb:32:in `perform'","lib/gitlab/sidekiq_middleware/concurrency_limit/server.rb:8:in `call'","lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'","lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'","lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'","lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'","lib/gitlab/application_context.rb:151:in `block in use'","lib/gitlab/application_context.rb:151:in `use'","lib/gitlab/sidekiq_middleware/worker_context.rb:11:in `wrap_in_optional_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'","lib/gitlab/application_context.rb:151:in `block in use'","lib/gitlab/application_context.rb:151:in `use'","lib/gitlab/application_context.rb:82:in `with_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'","lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'","lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'","lib/gitlab/database/query_analyzer.rb:40:in `within'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'","lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'","lib/gitlab/sidekiq_middleware/set_ip_address.rb:7:in `call'","lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'","lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'","lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `block in instrument'","lib/gitlab/metrics/background_transaction.rb:33:in `run'","lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `instrument'","lib/gitlab/sidekiq_middleware/server_metrics.rb:110:in `call'","lib/gitlab/query_limiting/sidekiq_middleware.rb:10:in `block in call'","lib/gitlab/query_limiting/transaction.rb:48:in `run'","lib/gitlab/query_limiting/sidekiq_middleware.rb:9:in `call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'","lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'","lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'","lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'","lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:10:in `block in call'","lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'","lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:9:in `call'","lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'","lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"],"db_duration_s":0.009075,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
Edited by Terri Chu

Merge request reports

Loading