Do not index associated issues that are epic work item type
What does this MR do and why?
Corrective action from gitlab-com/gl-infra/production#18330 (closed)
There are a few cases where associated records are queued for indexing using ElasticAssociationIndexerWorker. That completely bypasses the checks for issues that are work item type epic which cannot be indexed into the issues index (due to not having a project_id field).
This MR prevents associations from indexing by:
- adding a
searchablescope toissuemodel - apply
searchablescope toIssueclass records before queueing for indexing
MR acceptance checklist
Please evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.
Screenshots or screen recordings
N/A
How to set up and validate locally
- enable gdk for elasticsearch and index everything
- create issues from a "non-admin" user that are
epicwork item type issues - ban the non-admin user
- monitor the logs in
sidekiq.logandelasticsearch.logto ensure the record is not queued
on master, you would see the undefined method for nil class error but on this branch you should not
error from `master` branch
2024-07-25_14:54:13.56063 rails-background-jobs : {"severity":"WARN","time":"2024-07-25T14:54:13.560Z","retry":0,"queue":"default","backtrace":true,"version":0,"store":null,"queue_namespace":"cronjob","args":["1"],"class":"ElasticIndexBulkCronWorker","jid":"58d47dafe50ed286bee8dcc6","created_at":"2024-07-25T14:54:13.308Z","trace_propagation_headers":{"sentry-trace":"3778606643654bba839976194217ac73-0ab533a1498a4603","baggage":"sentry-trace_id=3778606643654bba839976194217ac73,sentry-environment=,sentry-release=2d9968d18a9"},"meta.caller_id":"Cronjob","correlation_id":"c7c05be76d30a73e5cec2cac3e0cb2c5","meta.root_caller_id":"Cronjob","meta.feature_category":"global_search","worker_data_consistency":"sticky","wal_locations":{},"wal_location_source":"primary","idempotency_key":"resque:gitlab:duplicate:default:e299b1e1b7aa4f7fa219749dc82aa084982849f63721e1d1c8ae48fe38f6530b","size_limiter":"validated","enqueued_at":"2024-07-25T14:54:13.311Z","job_size_bytes":3,"pid":64347,"message":"ElasticIndexBulkCronWorker JID-58d47dafe50ed286bee8dcc6: fail: 0.135342 sec","job_status":"fail","scheduling_latency_s":0.113117,"db_count":5,"db_write_count":0,"db_cached_count":0,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":5,"db_main_count":5,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_max_duration_s":0.0,"db_primary_txn_max_duration_s":0.0,"db_main_txn_max_duration_s":0.0,"db_ci_txn_max_duration_s":0.0,"db_main_replica_txn_max_duration_s":0.0,"db_ci_replica_txn_max_duration_s":0.0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.028,"db_main_duration_s":0.028,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.01412,"worker_id":"sidekiq_0","rate_limiting_gates":[],"exclusive_lock_requested_count":1,"exclusive_lock_wait_duration_s":0.001162,"exclusive_lock_hold_duration_s":0.021507,"duration_s":0.135342,"completed_at":"2024-07-25T14:54:13.560Z","load_balancing_strategy":"primary_no_wal","exception.class":"NoMethodError","exception.message":"undefined method `visibility_level' for nil:NilClass","exception.backtrace":["ee/lib/elastic/latest/issue_instance_proxy.rb:27:in `as_indexed_json'","ee/lib/elastic/multi_version_util.rb:76:in `as_indexed_json'","ee/lib/gitlab/elastic/document_reference.rb:125:in `as_indexed_json'","ee/lib/gitlab/elastic/bulk_indexer.rb:178:in `upsert_operation'","ee/lib/gitlab/elastic/bulk_indexer.rb:78:in `upsert'","ee/lib/gitlab/elastic/bulk_indexer.rb:46:in `process'","ee/app/services/elastic/process_bookkeeping_service.rb:267:in `submit_document'","ee/app/services/elastic/process_bookkeeping_service.rb:187:in `block in execute_with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:186:in `each'","ee/app/services/elastic/process_bookkeeping_service.rb:186:in `execute_with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:148:in `block in execute'","lib/gitlab/redis/wrapper.rb:25:in `block in with'","lib/gitlab/redis/wrapper.rb:25:in `with'","ee/app/services/elastic/process_bookkeeping_service.rb:106:in `with_redis'","ee/app/services/elastic/process_bookkeeping_service.rb:148:in `execute'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:50:in `block in process_shard'","lib/gitlab/exclusive_lease_helpers.rb:43:in `block in in_lock'","lib/gitlab/exclusive_lease_helpers.rb:53:in `with_instrumentation'","lib/gitlab/exclusive_lease_helpers.rb:42:in `in_lock'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:49:in `process_shard'","ee/app/workers/concerns/elastic/bulk_cron_worker.rb:34:in `perform'","ee/app/workers/concerns/geo/skip_secondary.rb:14:in `perform'","ee/lib/gitlab/sidekiq_middleware/set_session/server.rb:21:in `call'","lib/gitlab/sidekiq_middleware/skip_jobs.rb:51:in `call'","lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'","lib/gitlab/sidekiq_middleware/concurrency_limit/middleware.rb:32:in `perform'","lib/gitlab/sidekiq_middleware/concurrency_limit/server.rb:8:in `call'","lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'","lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'","lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'","lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'","lib/gitlab/application_context.rb:151:in `block in use'","lib/gitlab/application_context.rb:151:in `use'","lib/gitlab/sidekiq_middleware/worker_context.rb:11:in `wrap_in_optional_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'","lib/gitlab/application_context.rb:151:in `block in use'","lib/gitlab/application_context.rb:151:in `use'","lib/gitlab/application_context.rb:82:in `with_context'","lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'","lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'","lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'","lib/gitlab/database/query_analyzer.rb:40:in `within'","lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'","lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'","lib/gitlab/sidekiq_middleware/set_ip_address.rb:7:in `call'","lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'","lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'","lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `block in instrument'","lib/gitlab/metrics/background_transaction.rb:33:in `run'","lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `instrument'","lib/gitlab/sidekiq_middleware/server_metrics.rb:110:in `call'","lib/gitlab/query_limiting/sidekiq_middleware.rb:10:in `block in call'","lib/gitlab/query_limiting/transaction.rb:48:in `run'","lib/gitlab/query_limiting/sidekiq_middleware.rb:9:in `call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'","lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'","lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'","lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'","lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:10:in `block in call'","lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'","lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:9:in `call'","lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'","lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"],"db_duration_s":0.009075,"urgency":"low","target_duration_s":300,"target_scheduling_latency_s":60}
Edited by Terri Chu