Security Policy unassignment fails when there are more projects than can be processed inside RequestTimeoutException limit

Summary

Un-assigning Security Policies from a namespace can fail if there are enough operations to exceed the RequestTimeoutException

Steps to reproduce

1. Have an instance or namespace with a lot of projects
2. Unlink the security policies
3. Wait for RequestTimeoutException
4. See GUI error Unexpected token '<', "<!DOCTYPE "… is not valid JSON

Example Project

N/A

What is the current bug behavior?

Unlinking policies fails with a cryptic error

What is the expected correct behavior?

Unlinking policies should run until complete

Relevant logs and/or screenshots

{
  "_index": "pubsub-rails-inf-gprd-026130",
  "_id": "_bd515QBQiL5-D0SoTp9",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2025-02-05T18:57:39.540Z",
    "type": "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf",
    "kubernetes": {
      "pod_ip": "10.64.99.37",
      "zone": "us-east1-b",
      "region": "us-east1",
      "namespace_name": "gitlab",
      "container_image": "us-east1-docker.pkg.dev/gitlab-com-artifact-registry/images/gitlab-webservice-ee:17-9-202502051207-93b133a2655",
      "container_name": "webservice",
      "pod_name": "gitlab-webservice-api-59b589c4db-bhd29",
      "host": "gke-gprd-us-east1-b-generic-3-bc3a5739-bd99"
    },
    "publish_time": "2025-02-05T18:57:39.412Z",
    "json": {
      "remote_ip": "xxxx.xxx.xxx.xxx",
      "redis_db_load_balancing_write_bytes": 180,
      "redis_db_load_balancing_calls": 3,
      "db_ci_wal_count": 0,
      "db_main_replica_wal_count": 9138,
      "pid": 75,
      "redis_queues_metadata_duration_s": 4.525108,
      "db_ci_replica_txn_duration_s": 0,
      "redis_sessions_allowed_cross_slot_calls": 1,
      "db_main_replica_txn_count": 0,
      "db_main_txn_count": 0,
      "db_main_count": 1,
      "meta.caller_id": "GraphqlController#execute",
      "redis_queues_metadata_calls": 9139,
      "user_id": xxxx,
      "db_embedding_txn_max_duration_s": 0,
      "meta.user_id": xxxxx,
      "request_urgency": "low",
      "subcomponent": "production_json",
      "db_ci_replica_write_count": 0,
      "db_embedding_replica_wal_count": 0,
      "db_ci_count": 0,
      "cpu_s": 37.570696,
      "component": "gitlab",
      "redis_calls": 27421,
      "db_ci_txn_max_duration_s": 0,
      "db_main_wal_cached_count": 0,
      "cf_ipcountry": "US",
      "mem_mallocs": 4664855,
      "redis_rate_limiting_read_bytes": 2,
      "meta.client_id": "user/xxxxx",
      "db_main_replica_cached_count": 0,
      "db_ci_duration_s": 0,
      "meta.organization_id": 1,
      "db_embedding_wal_cached_count": 0,
      "graphql": [
        {
          "depth": 2,
          "used_fields": [
            "SecurityPolicyProjectUnassignPayload.clientMutationId",
            "SecurityPolicyProjectUnassignPayload.errors",
            "Mutation.securityPolicyProjectUnassign"
          ],
          "used_deprecated_fields": [],
          "used_deprecated_arguments": [],
          "variables": "{}",
          "operation_name": "securityPolicyProjectUnassign",
          "complexity": 3
        }
      ],
      "db_ci_txn_duration_s": 0,
      "method": "POST",
      "db_embedding_replica_txn_count": 0,
      "redis_sessions_read_bytes": 552,
      "redis_queues_catchall_a_duration_s": 4.686401,
      "db_main_replica_duration_s": 5.467,
      "redis_sessions_calls": 3,
      "db_ci_replica_duration_s": 5.354,
      "redis_write_bytes": 14876371,
      "redis_db_load_balancing_duration_s": 0.000826,
      "rate_limiting_gates": [],
      "db_main_cached_count": 0,
      "logtag": "F",
      "db_embedding_replica_count": 0,
      "redis_sessions_duration_s": 0.000856,
      "db_embedding_replica_wal_cached_count": 0,
      "db_main_replica_write_count": 0,
      "redis_queues_catchall_a_calls": 18274,
      "db_ci_replica_wal_count": 9137,
      "db_embedding_cached_count": 0,
      "db_embedding_replica_duration_s": 0,
      "tag": "api-rails.var.log.containers.gitlab-webservice-api-59b589c4db-bhd29_gitlab_webservice-db794b5244717e92d85d0091d3604d6bf084558831c96f5dc0e5d05edcada1ca.log",
      "db_main_write_count": 0,
      "environment": "gprd",
      "time": "2025-02-05T18:57:18.502Z",
      "redis_read_bytes": 37342,
      "db_embedding_replica_txn_duration_s": 0,
      "status": 500,
      "mem_objects": 14326594,
      "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
      "path": "/api/graphql",
      "db_embedding_write_count": 0,
      "target_duration_s": 5,
      "db_main_replica_txn_duration_s": 0,
      "db_embedding_txn_count": 0,
      "controller": "GraphqlController",
      "cf_worker": "gitlab.com",
      "db_ci_cached_count": 0,
      "cf_ray": "90d51ea35317a384-ATL",
      "meta.user": "[redacted]",
      "username": "[redacted]",
      "db_ci_replica_txn_max_duration_s": 0,
      "meta.remote_ip": "xxx.xxx.xxx.xxx",
      "mem_bytes": 608628208,
      "redis_queues_metadata_write_bytes": 2254920,
      "db_main_wal_count": 0,
      "view_duration_s": 0,
      "params": [
        {
          "key": "operationName",
          "value": "securityPolicyProjectUnassign"
        },
        {
          "key": "variables",
          "value": "[FILTERED]"
        },
        {
          "key": "query",
          "value": "[FILTERED]"
        },
        {
          "key": "graphql",
          "value": "{\"operationName\"=>\"securityPolicyProjectUnassign\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"[FILTERED]\"}"
        }
      ],
      "redis_feature_flag_read_bytes": 236,
      "redis_queues_catchall_a_write_bytes": 12620509,
      "stage": "main",
      "redis_queues_catchall_a_read_bytes": 18274,
      "db_embedding_txn_duration_s": 0,
      "db_duration_s": 10.16631,
      "type": "api",
      "tier": "sv",
      "queue_duration_s": 0.020119,
      "exception.message": "Request ran for longer than 60000ms ",
      "redis_rate_limiting_duration_s": 0.000265,
      "correlation_id": "a20324bd52c82b65a98249214d16b796",
      "db_ci_replica_count": 9137,
      "meta.feature_category": "not_owned",
      "db_ci_txn_count": 0,
      "redis_rate_limiting_calls": 1,
      "redis_allowed_cross_slot_calls": 1,
      "db_embedding_count": 0,
      "redis_feature_flag_duration_s": 0.00032,
      "db_embedding_replica_txn_max_duration_s": 0,
      "db_main_replica_count": 9159,
      "redis_feature_flag_calls": 1,
      "db_ci_replica_txn_count": 0,
      "worker_id": "puma_4",
      "db_main_replica_wal_cached_count": 0,
      "exception.backtrace": [
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `exec_params'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `block (2 levels) in exec_no_cache'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/share_lock.rb:187:in `yield_shares'",
        "activesupport (7.0.8.7) lib/active_support/dependencies/interlock.rb:41:in `permit_concurrent_loads'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:767:in `block in exec_no_cache'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:752:in `block in log'",
        "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:743:in `log'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:766:in `exec_no_cache'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:745:in `execute_and_clear'",
        "marginalia (1.11.1) lib/marginalia.rb:91:in `execute_and_clear_with_marginalia'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql/database_statements.rb:54:in `exec_query'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:560:in `select'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:66:in `select_all'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `block in select_all'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:137:in `block in cache_sql'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
        "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:128:in `cache_sql'",
        "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `select_all'",
        "lib/gitlab/database/load_balancing/host.rb:283:in `query_and_release_old'",
        "lib/gitlab/database/load_balancing/host.rb:278:in `query_and_release'",
        "lib/gitlab/database/load_balancing/host.rb:256:in `database_replica_location'",
        "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:24:in `wal_location_for'",
        "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:10:in `block (2 levels) in wal_locations_by_db_name'",
        "lib/gitlab/database/load_balancing.rb:29:in `block in each_load_balancer'",
        "lib/gitlab/database/load_balancing.rb:28:in `each'",
        "lib/gitlab/database/load_balancing.rb:28:in `each_load_balancer'",
        "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:9:in `block in wal_locations_by_db_name'",
        "<internal:kernel>:90:in `tap'",
        "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:8:in `wal_locations_by_db_name'",
        "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:38:in `set_data_consistency_locations!'",
        "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:19:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:21:in `block in call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:23:in `block in call'",
        "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:20:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:20:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
        "lib/gitlab/application_context.rb:173:in `block in use'",
        "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
        "lib/gitlab/application_context.rb:173:in `use'",
        "lib/gitlab/application_context.rb:96:in `with_context'",
        "lib/gitlab/sidekiq_middleware/worker_context/client.rb:36:in `block in call'",
        "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
        "lib/gitlab/sidekiq_middleware/worker_context/client.rb:18:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
        "lib/gitlab/sidekiq_middleware/concurrency_limit/resume.rb:17:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
        "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:103:in `block in call'",
        "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
        "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
        "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:100:in `call'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:88:in `push'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:361:in `client_push'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:198:in `perform_async'",
        "lib/gitlab/patch/sidekiq_job_setter.rb:34:in `block in perform_async'",
        "lib/gitlab/sidekiq_sharding/router.rb:37:in `block (2 levels) in route'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:167:in `via'",
        "lib/gitlab/sidekiq_sharding/validator.rb:17:in `via'",
        "lib/gitlab/sidekiq_sharding/router.rb:36:in `block in route'",
        "lib/gitlab/application_context.rb:173:in `block in use'",
        "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
        "lib/gitlab/application_context.rb:173:in `use'",
        "lib/gitlab/application_context.rb:96:in `with_context'",
        "lib/gitlab/sidekiq_sharding/router.rb:35:in `route'",
        "lib/gitlab/patch/sidekiq_job_setter.rb:32:in `perform_async'",
        "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:286:in `perform_async'",
        "config/initializers/forbid_sidekiq_in_transactions.rb:60:in `block (2 levels) in <module:NoEnqueueingFromTransactions>'",
        "ee/app/services/security/orchestration/unassign_service.rb:40:in `block in remove_bot'",
        "ee/app/services/security/orchestration/unassign_service.rb:39:in `each'",
        "ee/app/services/security/orchestration/unassign_service.rb:39:in `remove_bot'",
        "ee/app/services/security/orchestration/unassign_service.rb:11:in `execute'",
        "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:36:in `unassign'",
        "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:25:in `resolve'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `public_send'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `call_resolve'",
        "graphql (2.4.8) lib/graphql/schema/mutation.rb:69:in `call_resolve'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:108:in `block (3 levels) in resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:96:in `block (2 levels) in resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:87:in `block in resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/schema/resolver.rb:75:in `resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema/has_single_input_argument.rb:36:in `resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema/relay_classic_mutation.rb:44:in `resolve_with_support'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `public_send'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `block (2 levels) in resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:881:in `block in with_extensions'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:920:in `run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field_extension.rb:134:in `resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
        "lib/gitlab/graphql/present/field_extension.rb:18:in `resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:876:in `with_extensions'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:707:in `block in resolve'",
        "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/schema/field.rb:705:in `resolve'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:375:in `block (2 levels) in evaluate_selection_with_resolved_keyword_args'",
        "graphql (2.4.8) lib/graphql/tracing/trace.rb:55:in `execute_field'",
        "batch-loader (2.0.5) lib/batch_loader/graphql.rb:7:in `execute_field'",
        "graphql (2.4.8) lib/graphql/schema/timeout.rb:84:in `execute_field'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:374:in `block in evaluate_selection_with_resolved_keyword_args'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:363:in `evaluate_selection_with_resolved_keyword_args'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:341:in `block in evaluate_selection_with_args'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:810:in `after_lazy'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:288:in `evaluate_selection_with_args'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:282:in `block in evaluate_selection'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/arguments_cache.rb:41:in `dataload_for'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:280:in `evaluate_selection'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:217:in `block (3 levels) in evaluate_selections'",
        "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:13:in `run_isolated'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:216:in `block (2 levels) in evaluate_selections'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `each'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `block in evaluate_selections'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:206:in `evaluate_selections'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:96:in `block (3 levels) in run_eager'",
        "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:95:in `block (2 levels) in run_eager'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:116:in `each_gathered_selections'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:86:in `block in run_eager'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
        "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:85:in `run_eager'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:73:in `block (4 levels) in run_all'",
        "graphql (2.4.8) lib/graphql/tracing/trace.rb:47:in `execute_query'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:72:in `block (3 levels) in run_all'",
        "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:60:in `block (2 levels) in run_all'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each_with_index'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `block in run_all'",
        "graphql (2.4.8) lib/graphql/tracing/trace.rb:43:in `execute_multiplex'",
        "lib/gitlab/graphql/tracers/instrumentation_tracer.rb:23:in `execute_multiplex'",
        "graphql (2.4.8) lib/graphql/schema/timeout.rb:68:in `execute_multiplex'",
        "graphql (2.4.8) lib/graphql/execution/interpreter.rb:38:in `run_all'",
        "graphql (2.4.8) lib/graphql/schema.rb:1545:in `multiplex'",
        "app/graphql/gitlab_schema.rb:44:in `multiplex'",
        "graphql (2.4.8) lib/graphql/schema.rb:1521:in `execute'",
        "app/controllers/graphql_controller.rb:226:in `execute_query'",
        "app/controllers/graphql_controller.rb:63:in `execute'",
        "actionpack (7.0.8.7) lib/action_controller/metal/basic_implicit_render.rb:6:in `send_action'",
        "actionpack (7.0.8.7) lib/abstract_controller/base.rb:215:in `process_action'",
        "actionpack (7.0.8.7) lib/action_controller/metal/rendering.rb:165:in `process_action'",
        "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:234:in `block in process_action'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:118:in `block in run_callbacks'",
        "lib/gitlab/ip_address_state.rb:11:in `with'",
        "ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "app/controllers/application_controller.rb:491:in `set_current_admin'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "lib/gitlab/session.rb:11:in `with_session'",
        "app/controllers/application_controller.rb:482:in `set_session_storage'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "lib/gitlab/i18n.rb:114:in `with_locale'",
        "lib/gitlab/i18n.rb:120:in `with_user_locale'",
        "app/controllers/application_controller.rb:473:in `set_locale'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "app/controllers/application_controller.rb:466:in `set_current_context'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "marginalia (1.11.1) lib/marginalia.rb:109:in `record_query_comment'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:34:in `block in sentry_around_action'",
        "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
        "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
        "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:18:in `sentry_around_action'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:138:in `run_callbacks'",
        "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:233:in `process_action'",
        "actionpack (7.0.8.7) lib/action_controller/metal/rescue.rb:23:in `process_action'",
        "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:67:in `block in process_action'",
        "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `block in instrument'",
        "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
        "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `instrument'",
        "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:66:in `process_action'",
        "actionpack (7.0.8.7) lib/action_controller/metal/params_wrapper.rb:259:in `process_action'",
        "activerecord (7.0.8.7) lib/active_record/railties/controller_runtime.rb:27:in `process_action'",
        "actionpack (7.0.8.7) lib/abstract_controller/base.rb:151:in `process'",
        "actionview (7.0.8.7) lib/action_view/rendering.rb:39:in `process'",
        "actionpack (7.0.8.7) lib/action_controller/metal.rb:188:in `dispatch'",
        "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `block in dispatch'",
        "lib/gitlab/middleware/action_controller_static_context.rb:23:in `call'",
        "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `dispatch'",
        "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:49:in `dispatch'",
        "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:32:in `serve'",
        "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:50:in `block in serve'",
        "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `each'",
        "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `serve'",
        "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:852:in `call'",
        "gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:470:in `call_app!'",
        "ee/lib/omni_auth/strategies/group_saml.rb:41:in `other_phase'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:195:in `call!'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
        "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
        "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'",
        "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:37:in `call'",
        "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
        "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
        "lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
        "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
        "lib/gitlab/middleware/memory_report.rb:13:in `call'",
        "lib/gitlab/middleware/speedscope.rb:13:in `call'",
        "lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
        "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
        "lib/gitlab/etag_caching/middleware.rb:21:in `call'",
        "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
        "lib/gitlab/metrics/web_transaction.rb:46:in `run'",
        "lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
        "lib/gitlab/middleware/go.rb:21:in `call'",
        "lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
        "lib/gitlab/database/query_analyzer.rb:83:in `within'",
        "lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
        "lib/ci/job_token/middleware.rb:11:in `call'",
        "batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'",
        "rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'",
        "apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'",
        "lib/gitlab/middleware/multipart.rb:173:in `call'",
        "rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'",
        "warden (1.2.9) lib/warden/manager.rb:36:in `block in call'",
        "warden (1.2.9) lib/warden/manager.rb:34:in `catch'",
        "warden (1.2.9) lib/warden/manager.rb:34:in `call'",
        "rack-cors (2.0.2) lib/rack/cors.rb:102:in `call'",
        "rack (2.2.10) lib/rack/tempfile_reaper.rb:15:in `call'",
        "rack (2.2.10) lib/rack/etag.rb:27:in `call'",
        "rack (2.2.10) lib/rack/conditional_get.rb:40:in `call'",
        "rack (2.2.10) lib/rack/head.rb:12:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/http/permissions_policy.rb:38:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/http/content_security_policy.rb:39:in `call'",
        "lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
        "lib/gitlab/middleware/read_only.rb:18:in `call'",
        "lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
        "rack (2.2.10) lib/rack/session/abstract/id.rb:266:in `context'",
        "rack (2.2.10) lib/rack/session/abstract/id.rb:260:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/cookies.rb:704:in `call'",
        "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:27:in `block in call'",
        "activesupport (7.0.8.7) lib/active_support/callbacks.rb:99:in `run_callbacks'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:26:in `call'",
        "sentry-rails (5.22.1) lib/sentry/rails/rescued_exception_interceptor.rb:14:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/debug_exceptions.rb:28:in `call'",
        "lib/gitlab/middleware/path_traversal_check.rb:40:in `call'",
        "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
        "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:30:in `block (2 levels) in call'",
        "sentry-ruby (5.22.1) lib/sentry/hub.rb:269:in `with_session_tracking'",
        "sentry-ruby (5.22.1) lib/sentry-ruby.rb:416:in `with_session_tracking'",
        "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:21:in `block in call'",
        "sentry-ruby (5.22.1) lib/sentry/hub.rb:59:in `with_scope'",
        "sentry-ruby (5.22.1) lib/sentry-ruby.rb:396:in `with_scope'",
        "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:20:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/show_exceptions.rb:29:in `call'",
        "lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
        "lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'",
        "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `block in call'",
        "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `block in tagged'",
        "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:37:in `tagged'",
        "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `tagged'",
        "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/remote_ip.rb:93:in `call'",
        "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
        "lib/gitlab/middleware/request_context.rb:15:in `call'",
        "lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
        "request_store (1.5.1) lib/request_store/middleware.rb:19:in `call'",
        "rack (2.2.10) lib/rack/method_override.rb:24:in `call'",
        "rack (2.2.10) lib/rack/runtime.rb:22:in `call'",
        "rack-timeout (0.7.0) lib/rack/timeout/core.rb:154:in `block in call'",
        "rack-timeout (0.7.0) lib/rack/timeout/support/timeout.rb:19:in `timeout'",
        "rack-timeout (0.7.0) lib/rack/timeout/core.rb:153:in `call'",
        "config/initializers/fix_local_cache_middleware.rb:11:in `call'",
        "lib/gitlab/middleware/compressed_json.rb:44:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/executor.rb:14:in `call'",
        "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
        "rack (2.2.10) lib/rack/sendfile.rb:110:in `call'",
        "lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
        "lib/gitlab/metrics/requests_rack_middleware.rb:83:in `call'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:22:in `block in call'",
        "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
        "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:21:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/request_id.rb:26:in `call'",
        "actionpack (7.0.8.7) lib/action_dispatch/middleware/host_authorization.rb:131:in `call'",
        "railties (7.0.8.7) lib/rails/engine.rb:530:in `call'",
        "railties (7.0.8.7) lib/rails/railtie.rb:226:in `public_send'",
        "railties (7.0.8.7) lib/rails/railtie.rb:226:in `method_missing'",
        "lib/gitlab/middleware/release_env.rb:12:in `call'",
        "rack (2.2.10) lib/rack/urlmap.rb:74:in `block in call'",
        "rack (2.2.10) lib/rack/urlmap.rb:58:in `each'",
        "rack (2.2.10) lib/rack/urlmap.rb:58:in `call'",
        "puma (6.5.0) lib/puma/configuration.rb:279:in `call'",
        "puma (6.5.0) lib/puma/request.rb:99:in `block in handle_request'",
        "puma (6.5.0) lib/puma/thread_pool.rb:389:in `with_force_shutdown'",
        "puma (6.5.0) lib/puma/request.rb:98:in `handle_request'",
        "puma (6.5.0) lib/puma/server.rb:468:in `process_client'",
        "puma (6.5.0) lib/puma/server.rb:249:in `block in run'",
        "puma (6.5.0) lib/puma/thread_pool.rb:166:in `block in spawn_thread'"
      ],
      "db_ci_write_count": 0,
      "db_ci_replica_wal_cached_count": 0,
      "redis_duration_s": 9.213776000000001,
      "meta.http_router_rule_action": "proxy",
      "db_embedding_duration_s": 0,
      "db_embedding_replica_cached_count": 0,
      "redis_rate_limiting_write_bytes": 78,
      "db_ci_wal_cached_count": 0,
      "exception.class": "Rack::Timeout::RequestTimeoutException",
      "path_traversal_check_duration_s": 0.000039,
      "db_ci_replica_cached_count": 0,
      "db_embedding_wal_count": 0,
      "db_main_replica_txn_max_duration_s": 0,
      "db_main_duration_s": 0.001,
      "duration_s": 59.98277,
      "shard": "default",
      "db_main_txn_max_duration_s": 0,
      "db_main_txn_duration_s": 0,
      "format": "*/*",
      "mem_total_bytes": 1181691968,
      "db_embedding_replica_write_count": 0,
      "redis_feature_flag_write_bytes": 63,
      "redis_sessions_write_bytes": 621,
      "action": "execute",
      "redis_queues_metadata_read_bytes": 18278
    },
    "host": {
      "name": "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf"
    }
  },
  "fields": {
    "json.meta.organization_id": [
      1
    ],
    "json.redis_sessions_allowed_cross_slot_calls": [
      1
    ],
    "host.name.keyword": [
      "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf"
    ],
    "json.db_embedding_duration_s": [
      0
    ],
    "json.db_main_wal_count": [
      0
    ],
    "json.logtag.keyword": [
      "F"
    ],
    "json.redis_queues_catchall_a_write_bytes": [
      12620509
    ],
    "json.exception.class.keyword": [
      "Rack::Timeout::RequestTimeoutException"
    ],
    "json.db_embedding_replica_count": [
      0
    ],
    "json.db_embedding_replica_txn_max_duration_s": [
      0
    ],
    "json.redis_db_load_balancing_write_bytes": [
      180
    ],
    "json.params.value": [
      "securityPolicyProjectUnassign",
      "[FILTERED]",
      "[FILTERED]",
      "{\"operationName\"=>\"securityPolicyProjectUnassign\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"[FILTERED]\"}"
    ],
    "json.cf_ray": [
      "90d51ea35317a384-ATL"
    ],
    "json.target_duration_s": [
      5
    ],
    "json.meta.caller_id": [
      "GraphqlController#execute"
    ],
    "json.subcomponent.keyword": [
      "production_json"
    ],
    "json.db_ci_replica_txn_count": [
      0
    ],
    "json.db_embedding_replica_txn_duration_s": [
      0
    ],
    "json.exception.message.keyword": [
      "Request ran for longer than 60000ms "
    ],
    "json.db_ci_replica_count": [
      9137
    ],
    "json.user_id": [
      "xxxxxx"
    ],
    "json.db_ci_replica_write_count": [
      0
    ],
    "json.db_main_replica_txn_count": [
      0
    ],
    "json.environment": [
      "gprd"
    ],
    "json.graphql.operation_name.keyword": [
      "securityPolicyProjectUnassign"
    ],
    "json.redis_feature_flag_read_bytes": [
      236
    ],
    "json.db_embedding_wal_count": [
      0
    ],
    "json.db_main_replica_write_count": [
      0
    ],
    "kubernetes.pod_ip": [
      "10.64.99.37"
    ],
    "json.db_embedding_replica_wal_count": [
      0
    ],
    "json.params.value.keyword": [
      "securityPolicyProjectUnassign",
      "[FILTERED]",
      "[FILTERED]",
      "{\"operationName\"=>\"securityPolicyProjectUnassign\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"[FILTERED]\"}"
    ],
    "json.type": [
      "api"
    ],
    "json.meta.feature_category": [
      "not_owned"
    ],
    "json.db_embedding_txn_max_duration_s": [
      0
    ],
    "json.worker_id.keyword": [
      "puma_4"
    ],
    "json.duration_s": [
      59.98277
    ],
    "json.path_traversal_check_duration_s": [
      0.000039
    ],
    "kubernetes.namespace_name.keyword": [
      "gitlab"
    ],
    "json.cf_worker": [
      "gitlab.com"
    ],
    "json.db_main_replica_txn_duration_s": [
      0
    ],
    "json.format": [
      "*/*"
    ],
    "json.ua.keyword": [
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0"
    ],
    "json.worker_id": [
      "puma_4"
    ],
    "json.db_ci_duration_s": [
      0
    ],
    "kubernetes.pod_name": [
      "gitlab-webservice-api-59b589c4db-bhd29"
    ],
    "json.db_main_count": [
      1
    ],
    "json.graphql.complexity": [
      3
    ],
    "json.meta.http_router_rule_action": [
      "proxy"
    ],
    "json.db_embedding_replica_cached_count": [
      0
    ],
    "json.redis_rate_limiting_calls": [
      1
    ],
    "kubernetes.container_image.keyword": [
      "us-east1-docker.pkg.dev/gitlab-com-artifact-registry/images/gitlab-webservice-ee:17-9-202502051207-93b133a2655"
    ],
    "type.keyword": [
      "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf"
    ],
    "json.db_embedding_txn_duration_s": [
      0
    ],
    "kubernetes.container_image": [
      "us-east1-docker.pkg.dev/gitlab-com-artifact-registry/images/gitlab-webservice-ee:17-9-202502051207-93b133a2655"
    ],
    "json.db_embedding_count": [
      0
    ],
    "json.db_ci_wal_cached_count": [
      0
    ],
    "kubernetes.host.keyword": [
      "gke-gprd-us-east1-b-generic-3-bc3a5739-bd99"
    ],
    "json.graphql.variables": [
      "{}"
    ],
    "json.redis_sessions_calls": [
      3
    ],
    "json.tier.keyword": [
      "sv"
    ],
    "json.redis_queues_metadata_write_bytes": [
      2254920
    ],
    "json.redis_rate_limiting_read_bytes": [
      2
    ],
    "json.db_main_txn_max_duration_s": [
      0
    ],
    "json.cf_worker.keyword": [
      "gitlab.com"
    ],
    "kubernetes.container_name.keyword": [
      "webservice"
    ],
    "json.ua": [
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0"
    ],
    "json.exception.backtrace.keyword": [
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `exec_params'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `block (2 levels) in exec_no_cache'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/share_lock.rb:187:in `yield_shares'",
      "activesupport (7.0.8.7) lib/active_support/dependencies/interlock.rb:41:in `permit_concurrent_loads'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:767:in `block in exec_no_cache'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:752:in `block in log'",
      "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:743:in `log'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:766:in `exec_no_cache'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:745:in `execute_and_clear'",
      "marginalia (1.11.1) lib/marginalia.rb:91:in `execute_and_clear_with_marginalia'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql/database_statements.rb:54:in `exec_query'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:560:in `select'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:66:in `select_all'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `block in select_all'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:137:in `block in cache_sql'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:128:in `cache_sql'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `select_all'",
      "lib/gitlab/database/load_balancing/host.rb:283:in `query_and_release_old'",
      "lib/gitlab/database/load_balancing/host.rb:278:in `query_and_release'",
      "lib/gitlab/database/load_balancing/host.rb:256:in `database_replica_location'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:24:in `wal_location_for'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:10:in `block (2 levels) in wal_locations_by_db_name'",
      "lib/gitlab/database/load_balancing.rb:29:in `block in each_load_balancer'",
      "lib/gitlab/database/load_balancing.rb:28:in `each'",
      "lib/gitlab/database/load_balancing.rb:28:in `each_load_balancer'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:9:in `block in wal_locations_by_db_name'",
      "<internal:kernel>:90:in `tap'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:8:in `wal_locations_by_db_name'",
      "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:38:in `set_data_consistency_locations!'",
      "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:19:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:21:in `block in call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:23:in `block in call'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:20:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:20:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "lib/gitlab/application_context.rb:173:in `block in use'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "lib/gitlab/application_context.rb:173:in `use'",
      "lib/gitlab/application_context.rb:96:in `with_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/client.rb:36:in `block in call'",
      "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/client.rb:18:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "lib/gitlab/sidekiq_middleware/concurrency_limit/resume.rb:17:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:103:in `block in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
      "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:100:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:88:in `push'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:361:in `client_push'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:198:in `perform_async'",
      "lib/gitlab/patch/sidekiq_job_setter.rb:34:in `block in perform_async'",
      "lib/gitlab/sidekiq_sharding/router.rb:37:in `block (2 levels) in route'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:167:in `via'",
      "lib/gitlab/sidekiq_sharding/validator.rb:17:in `via'",
      "lib/gitlab/sidekiq_sharding/router.rb:36:in `block in route'",
      "lib/gitlab/application_context.rb:173:in `block in use'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "lib/gitlab/application_context.rb:173:in `use'",
      "lib/gitlab/application_context.rb:96:in `with_context'",
      "lib/gitlab/sidekiq_sharding/router.rb:35:in `route'",
      "lib/gitlab/patch/sidekiq_job_setter.rb:32:in `perform_async'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:286:in `perform_async'",
      "config/initializers/forbid_sidekiq_in_transactions.rb:60:in `block (2 levels) in <module:NoEnqueueingFromTransactions>'",
      "ee/app/services/security/orchestration/unassign_service.rb:40:in `block in remove_bot'",
      "ee/app/services/security/orchestration/unassign_service.rb:39:in `each'",
      "ee/app/services/security/orchestration/unassign_service.rb:39:in `remove_bot'",
      "ee/app/services/security/orchestration/unassign_service.rb:11:in `execute'",
      "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:36:in `unassign'",
      "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:25:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `public_send'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `call_resolve'",
      "graphql (2.4.8) lib/graphql/schema/mutation.rb:69:in `call_resolve'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:108:in `block (3 levels) in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:96:in `block (2 levels) in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:87:in `block in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:75:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/has_single_input_argument.rb:36:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/relay_classic_mutation.rb:44:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `public_send'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `block (2 levels) in resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:881:in `block in with_extensions'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:920:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field_extension.rb:134:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
      "lib/gitlab/graphql/present/field_extension.rb:18:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:876:in `with_extensions'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:707:in `block in resolve'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:705:in `resolve'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:375:in `block (2 levels) in evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:55:in `execute_field'",
      "batch-loader (2.0.5) lib/batch_loader/graphql.rb:7:in `execute_field'",
      "graphql (2.4.8) lib/graphql/schema/timeout.rb:84:in `execute_field'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:374:in `block in evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:363:in `evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:341:in `block in evaluate_selection_with_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:810:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:288:in `evaluate_selection_with_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:282:in `block in evaluate_selection'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/arguments_cache.rb:41:in `dataload_for'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:280:in `evaluate_selection'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:217:in `block (3 levels) in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:13:in `run_isolated'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:216:in `block (2 levels) in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `each'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `block in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:206:in `evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:96:in `block (3 levels) in run_eager'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:95:in `block (2 levels) in run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:116:in `each_gathered_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:86:in `block in run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:85:in `run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:73:in `block (4 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:47:in `execute_query'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:72:in `block (3 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:60:in `block (2 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each_with_index'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `block in run_all'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:43:in `execute_multiplex'",
      "lib/gitlab/graphql/tracers/instrumentation_tracer.rb:23:in `execute_multiplex'",
      "graphql (2.4.8) lib/graphql/schema/timeout.rb:68:in `execute_multiplex'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:38:in `run_all'",
      "graphql (2.4.8) lib/graphql/schema.rb:1545:in `multiplex'",
      "app/graphql/gitlab_schema.rb:44:in `multiplex'",
      "graphql (2.4.8) lib/graphql/schema.rb:1521:in `execute'",
      "app/controllers/graphql_controller.rb:226:in `execute_query'",
      "app/controllers/graphql_controller.rb:63:in `execute'",
      "actionpack (7.0.8.7) lib/action_controller/metal/basic_implicit_render.rb:6:in `send_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/base.rb:215:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/rendering.rb:165:in `process_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:234:in `block in process_action'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:118:in `block in run_callbacks'",
      "lib/gitlab/ip_address_state.rb:11:in `with'",
      "ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "app/controllers/application_controller.rb:491:in `set_current_admin'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "lib/gitlab/session.rb:11:in `with_session'",
      "app/controllers/application_controller.rb:482:in `set_session_storage'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "lib/gitlab/i18n.rb:114:in `with_locale'",
      "lib/gitlab/i18n.rb:120:in `with_user_locale'",
      "app/controllers/application_controller.rb:473:in `set_locale'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "app/controllers/application_controller.rb:466:in `set_current_context'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "marginalia (1.11.1) lib/marginalia.rb:109:in `record_query_comment'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:34:in `block in sentry_around_action'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
      "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:18:in `sentry_around_action'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:138:in `run_callbacks'",
      "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:233:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/rescue.rb:23:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:67:in `block in process_action'",
      "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `block in instrument'",
      "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
      "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `instrument'",
      "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:66:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/params_wrapper.rb:259:in `process_action'",
      "activerecord (7.0.8.7) lib/active_record/railties/controller_runtime.rb:27:in `process_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/base.rb:151:in `process'",
      "actionview (7.0.8.7) lib/action_view/rendering.rb:39:in `process'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:188:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `block in dispatch'",
      "lib/gitlab/middleware/action_controller_static_context.rb:23:in `call'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:49:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:32:in `serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:50:in `block in serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `each'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:852:in `call'",
      "gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:470:in `call_app!'",
      "ee/lib/omni_auth/strategies/group_saml.rb:41:in `other_phase'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:195:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'",
      "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:37:in `call'",
      "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
      "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
      "lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
      "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
      "lib/gitlab/middleware/memory_report.rb:13:in `call'",
      "lib/gitlab/middleware/speedscope.rb:13:in `call'",
      "lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
      "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
      "lib/gitlab/etag_caching/middleware.rb:21:in `call'",
      "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
      "lib/gitlab/metrics/web_transaction.rb:46:in `run'",
      "lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
      "lib/gitlab/middleware/go.rb:21:in `call'",
      "lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
      "lib/gitlab/database/query_analyzer.rb:83:in `within'",
      "lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
      "lib/ci/job_token/middleware.rb:11:in `call'",
      "batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'",
      "rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'",
      "apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'",
      "lib/gitlab/middleware/multipart.rb:173:in `call'",
      "rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'",
      "warden (1.2.9) lib/warden/manager.rb:36:in `block in call'",
      "warden (1.2.9) lib/warden/manager.rb:34:in `catch'",
      "warden (1.2.9) lib/warden/manager.rb:34:in `call'",
      "rack-cors (2.0.2) lib/rack/cors.rb:102:in `call'",
      "rack (2.2.10) lib/rack/tempfile_reaper.rb:15:in `call'",
      "rack (2.2.10) lib/rack/etag.rb:27:in `call'",
      "rack (2.2.10) lib/rack/conditional_get.rb:40:in `call'",
      "rack (2.2.10) lib/rack/head.rb:12:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/http/permissions_policy.rb:38:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/http/content_security_policy.rb:39:in `call'",
      "lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
      "lib/gitlab/middleware/read_only.rb:18:in `call'",
      "lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
      "rack (2.2.10) lib/rack/session/abstract/id.rb:266:in `context'",
      "rack (2.2.10) lib/rack/session/abstract/id.rb:260:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/cookies.rb:704:in `call'",
      "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:27:in `block in call'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:99:in `run_callbacks'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:26:in `call'",
      "sentry-rails (5.22.1) lib/sentry/rails/rescued_exception_interceptor.rb:14:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/debug_exceptions.rb:28:in `call'",
      "lib/gitlab/middleware/path_traversal_check.rb:40:in `call'",
      "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:30:in `block (2 levels) in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:269:in `with_session_tracking'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:416:in `with_session_tracking'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:21:in `block in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:59:in `with_scope'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:396:in `with_scope'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:20:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/show_exceptions.rb:29:in `call'",
      "lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
      "lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'",
      "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `block in call'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `block in tagged'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:37:in `tagged'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `tagged'",
      "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/remote_ip.rb:93:in `call'",
      "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
      "lib/gitlab/middleware/request_context.rb:15:in `call'",
      "lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
      "request_store (1.5.1) lib/request_store/middleware.rb:19:in `call'",
      "rack (2.2.10) lib/rack/method_override.rb:24:in `call'",
      "rack (2.2.10) lib/rack/runtime.rb:22:in `call'",
      "rack-timeout (0.7.0) lib/rack/timeout/core.rb:154:in `block in call'",
      "rack-timeout (0.7.0) lib/rack/timeout/support/timeout.rb:19:in `timeout'",
      "rack-timeout (0.7.0) lib/rack/timeout/core.rb:153:in `call'",
      "config/initializers/fix_local_cache_middleware.rb:11:in `call'",
      "lib/gitlab/middleware/compressed_json.rb:44:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/executor.rb:14:in `call'",
      "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
      "rack (2.2.10) lib/rack/sendfile.rb:110:in `call'",
      "lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
      "lib/gitlab/metrics/requests_rack_middleware.rb:83:in `call'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:22:in `block in call'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:21:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/request_id.rb:26:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/host_authorization.rb:131:in `call'",
      "railties (7.0.8.7) lib/rails/engine.rb:530:in `call'",
      "railties (7.0.8.7) lib/rails/railtie.rb:226:in `public_send'",
      "railties (7.0.8.7) lib/rails/railtie.rb:226:in `method_missing'",
      "lib/gitlab/middleware/release_env.rb:12:in `call'",
      "rack (2.2.10) lib/rack/urlmap.rb:74:in `block in call'",
      "rack (2.2.10) lib/rack/urlmap.rb:58:in `each'",
      "rack (2.2.10) lib/rack/urlmap.rb:58:in `call'",
      "puma (6.5.0) lib/puma/configuration.rb:279:in `call'",
      "puma (6.5.0) lib/puma/request.rb:99:in `block in handle_request'",
      "puma (6.5.0) lib/puma/thread_pool.rb:389:in `with_force_shutdown'",
      "puma (6.5.0) lib/puma/request.rb:98:in `handle_request'",
      "puma (6.5.0) lib/puma/server.rb:468:in `process_client'",
      "puma (6.5.0) lib/puma/server.rb:249:in `block in run'",
      "puma (6.5.0) lib/puma/thread_pool.rb:166:in `block in spawn_thread'"
    ],
    "json.remote_ip": [
      "xxx.xxx.xxx.xxx"
    ],
    "kubernetes.region": [
      "us-east1"
    ],
    "json.db_ci_write_count": [
      0
    ],
    "json.meta.caller_id.keyword": [
      "GraphqlController#execute"
    ],
    "json.db_embedding_cached_count": [
      0
    ],
    "json.graphql.depth": [
      2
    ],
    "json.params.key.keyword": [
      "operationName",
      "variables",
      "query",
      "graphql"
    ],
    "json.time": [
      "2025-02-05T18:57:18.502Z"
    ],
    "kubernetes.pod_name.keyword": [
      "gitlab-webservice-api-59b589c4db-bhd29"
    ],
    "json.exception.backtrace": [
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `exec_params'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:768:in `block (2 levels) in exec_no_cache'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/share_lock.rb:187:in `yield_shares'",
      "activesupport (7.0.8.7) lib/active_support/dependencies/interlock.rb:41:in `permit_concurrent_loads'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:767:in `block in exec_no_cache'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:752:in `block in log'",
      "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract_adapter.rb:743:in `log'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:766:in `exec_no_cache'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql_adapter.rb:745:in `execute_and_clear'",
      "marginalia (1.11.1) lib/marginalia.rb:91:in `execute_and_clear_with_marginalia'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/postgresql/database_statements.rb:54:in `exec_query'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:560:in `select'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/database_statements.rb:66:in `select_all'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `block in select_all'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:137:in `block in cache_sql'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'",
      "activesupport (7.0.8.7) lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:128:in `cache_sql'",
      "activerecord (7.0.8.7) lib/active_record/connection_adapters/abstract/query_cache.rb:107:in `select_all'",
      "lib/gitlab/database/load_balancing/host.rb:283:in `query_and_release_old'",
      "lib/gitlab/database/load_balancing/host.rb:278:in `query_and_release'",
      "lib/gitlab/database/load_balancing/host.rb:256:in `database_replica_location'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:24:in `wal_location_for'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:10:in `block (2 levels) in wal_locations_by_db_name'",
      "lib/gitlab/database/load_balancing.rb:29:in `block in each_load_balancer'",
      "lib/gitlab/database/load_balancing.rb:28:in `each'",
      "lib/gitlab/database/load_balancing.rb:28:in `each_load_balancer'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:9:in `block in wal_locations_by_db_name'",
      "<internal:kernel>:90:in `tap'",
      "lib/gitlab/database/load_balancing/wal_tracking_sender.rb:8:in `wal_locations_by_db_name'",
      "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:38:in `set_data_consistency_locations!'",
      "lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb:19:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:21:in `block in call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:23:in `block in call'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/client.rb:20:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/client.rb:20:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "lib/gitlab/application_context.rb:173:in `block in use'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "lib/gitlab/application_context.rb:173:in `use'",
      "lib/gitlab/application_context.rb:96:in `with_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/client.rb:36:in `block in call'",
      "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/client.rb:18:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "lib/gitlab/sidekiq_middleware/concurrency_limit/resume.rb:17:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
      "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:103:in `block in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
      "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:100:in `call'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:88:in `push'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:361:in `client_push'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:198:in `perform_async'",
      "lib/gitlab/patch/sidekiq_job_setter.rb:34:in `block in perform_async'",
      "lib/gitlab/sidekiq_sharding/router.rb:37:in `block (2 levels) in route'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/client.rb:167:in `via'",
      "lib/gitlab/sidekiq_sharding/validator.rb:17:in `via'",
      "lib/gitlab/sidekiq_sharding/router.rb:36:in `block in route'",
      "lib/gitlab/application_context.rb:173:in `block in use'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "lib/gitlab/application_context.rb:173:in `use'",
      "lib/gitlab/application_context.rb:96:in `with_context'",
      "lib/gitlab/sidekiq_sharding/router.rb:35:in `route'",
      "lib/gitlab/patch/sidekiq_job_setter.rb:32:in `perform_async'",
      "vendor/gems/sidekiq-7.2.4/lib/sidekiq/job.rb:286:in `perform_async'",
      "config/initializers/forbid_sidekiq_in_transactions.rb:60:in `block (2 levels) in <module:NoEnqueueingFromTransactions>'",
      "ee/app/services/security/orchestration/unassign_service.rb:40:in `block in remove_bot'",
      "ee/app/services/security/orchestration/unassign_service.rb:39:in `each'",
      "ee/app/services/security/orchestration/unassign_service.rb:39:in `remove_bot'",
      "ee/app/services/security/orchestration/unassign_service.rb:11:in `execute'",
      "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:36:in `unassign'",
      "ee/app/graphql/mutations/security_policy/unassign_security_policy_project.rb:25:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `public_send'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:121:in `call_resolve'",
      "graphql (2.4.8) lib/graphql/schema/mutation.rb:69:in `call_resolve'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:108:in `block (3 levels) in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:96:in `block (2 levels) in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:87:in `block in resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/resolver.rb:75:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/has_single_input_argument.rb:36:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/relay_classic_mutation.rb:44:in `resolve_with_support'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `public_send'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:736:in `block (2 levels) in resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:881:in `block in with_extensions'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block (2 levels) in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:920:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field_extension.rb:134:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:917:in `block in run_extensions_before_resolve'",
      "lib/gitlab/graphql/present/field_extension.rb:18:in `resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:904:in `run_extensions_before_resolve'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:876:in `with_extensions'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:707:in `block in resolve'",
      "graphql (2.4.8) lib/graphql/schema.rb:1589:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/query.rb:428:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/schema/field.rb:705:in `resolve'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:375:in `block (2 levels) in evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:55:in `execute_field'",
      "batch-loader (2.0.5) lib/batch_loader/graphql.rb:7:in `execute_field'",
      "graphql (2.4.8) lib/graphql/schema/timeout.rb:84:in `execute_field'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:374:in `block in evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:363:in `evaluate_selection_with_resolved_keyword_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:341:in `block in evaluate_selection_with_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:810:in `after_lazy'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:288:in `evaluate_selection_with_args'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:282:in `block in evaluate_selection'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/arguments_cache.rb:41:in `dataload_for'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:280:in `evaluate_selection'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:217:in `block (3 levels) in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:13:in `run_isolated'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:216:in `block (2 levels) in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `each'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:209:in `block in evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:206:in `evaluate_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:96:in `block (3 levels) in run_eager'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:95:in `block (2 levels) in run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:116:in `each_gathered_selections'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:86:in `block in run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:696:in `call_method_on_directives'",
      "graphql (2.4.8) lib/graphql/execution/interpreter/runtime.rb:85:in `run_eager'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:73:in `block (4 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:47:in `execute_query'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:72:in `block (3 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/dataloader/null_dataloader.rb:19:in `append_job'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:60:in `block (2 levels) in run_all'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `each_with_index'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:54:in `block in run_all'",
      "graphql (2.4.8) lib/graphql/tracing/trace.rb:43:in `execute_multiplex'",
      "lib/gitlab/graphql/tracers/instrumentation_tracer.rb:23:in `execute_multiplex'",
      "graphql (2.4.8) lib/graphql/schema/timeout.rb:68:in `execute_multiplex'",
      "graphql (2.4.8) lib/graphql/execution/interpreter.rb:38:in `run_all'",
      "graphql (2.4.8) lib/graphql/schema.rb:1545:in `multiplex'",
      "app/graphql/gitlab_schema.rb:44:in `multiplex'",
      "graphql (2.4.8) lib/graphql/schema.rb:1521:in `execute'",
      "app/controllers/graphql_controller.rb:226:in `execute_query'",
      "app/controllers/graphql_controller.rb:63:in `execute'",
      "actionpack (7.0.8.7) lib/action_controller/metal/basic_implicit_render.rb:6:in `send_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/base.rb:215:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/rendering.rb:165:in `process_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:234:in `block in process_action'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:118:in `block in run_callbacks'",
      "lib/gitlab/ip_address_state.rb:11:in `with'",
      "ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "app/controllers/application_controller.rb:491:in `set_current_admin'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "lib/gitlab/session.rb:11:in `with_session'",
      "app/controllers/application_controller.rb:482:in `set_session_storage'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "lib/gitlab/i18n.rb:114:in `with_locale'",
      "lib/gitlab/i18n.rb:120:in `with_user_locale'",
      "app/controllers/application_controller.rb:473:in `set_locale'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "app/controllers/application_controller.rb:466:in `set_current_context'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "marginalia (1.11.1) lib/marginalia.rb:109:in `record_query_comment'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:34:in `block in sentry_around_action'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:108:in `with_child_span'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:503:in `with_child_span'",
      "sentry-rails (5.22.1) lib/sentry/rails/controller_transaction.rb:18:in `sentry_around_action'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:127:in `block in run_callbacks'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:138:in `run_callbacks'",
      "actionpack (7.0.8.7) lib/abstract_controller/callbacks.rb:233:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/rescue.rb:23:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:67:in `block in process_action'",
      "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `block in instrument'",
      "activesupport (7.0.8.7) lib/active_support/notifications/instrumenter.rb:24:in `instrument'",
      "activesupport (7.0.8.7) lib/active_support/notifications.rb:206:in `instrument'",
      "actionpack (7.0.8.7) lib/action_controller/metal/instrumentation.rb:66:in `process_action'",
      "actionpack (7.0.8.7) lib/action_controller/metal/params_wrapper.rb:259:in `process_action'",
      "activerecord (7.0.8.7) lib/active_record/railties/controller_runtime.rb:27:in `process_action'",
      "actionpack (7.0.8.7) lib/abstract_controller/base.rb:151:in `process'",
      "actionview (7.0.8.7) lib/action_view/rendering.rb:39:in `process'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:188:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `block in dispatch'",
      "lib/gitlab/middleware/action_controller_static_context.rb:23:in `call'",
      "actionpack (7.0.8.7) lib/action_controller/metal.rb:249:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:49:in `dispatch'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:32:in `serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:50:in `block in serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `each'",
      "actionpack (7.0.8.7) lib/action_dispatch/journey/router.rb:32:in `serve'",
      "actionpack (7.0.8.7) lib/action_dispatch/routing/route_set.rb:852:in `call'",
      "gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:470:in `call_app!'",
      "ee/lib/omni_auth/strategies/group_saml.rb:41:in `other_phase'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:195:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:202:in `call!'",
      "omniauth (2.1.2) lib/omniauth/strategy.rb:169:in `call'",
      "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'",
      "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:37:in `call'",
      "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
      "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
      "lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
      "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
      "lib/gitlab/middleware/memory_report.rb:13:in `call'",
      "lib/gitlab/middleware/speedscope.rb:13:in `call'",
      "lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
      "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
      "lib/gitlab/etag_caching/middleware.rb:21:in `call'",
      "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
      "lib/gitlab/metrics/web_transaction.rb:46:in `run'",
      "lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
      "lib/gitlab/middleware/go.rb:21:in `call'",
      "lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
      "lib/gitlab/database/query_analyzer.rb:83:in `within'",
      "lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
      "lib/ci/job_token/middleware.rb:11:in `call'",
      "batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'",
      "rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'",
      "apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'",
      "lib/gitlab/middleware/multipart.rb:173:in `call'",
      "rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'",
      "warden (1.2.9) lib/warden/manager.rb:36:in `block in call'",
      "warden (1.2.9) lib/warden/manager.rb:34:in `catch'",
      "warden (1.2.9) lib/warden/manager.rb:34:in `call'",
      "rack-cors (2.0.2) lib/rack/cors.rb:102:in `call'",
      "rack (2.2.10) lib/rack/tempfile_reaper.rb:15:in `call'",
      "rack (2.2.10) lib/rack/etag.rb:27:in `call'",
      "rack (2.2.10) lib/rack/conditional_get.rb:40:in `call'",
      "rack (2.2.10) lib/rack/head.rb:12:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/http/permissions_policy.rb:38:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/http/content_security_policy.rb:39:in `call'",
      "lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
      "lib/gitlab/middleware/read_only.rb:18:in `call'",
      "lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
      "rack (2.2.10) lib/rack/session/abstract/id.rb:266:in `context'",
      "rack (2.2.10) lib/rack/session/abstract/id.rb:260:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/cookies.rb:704:in `call'",
      "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:27:in `block in call'",
      "activesupport (7.0.8.7) lib/active_support/callbacks.rb:99:in `run_callbacks'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/callbacks.rb:26:in `call'",
      "sentry-rails (5.22.1) lib/sentry/rails/rescued_exception_interceptor.rb:14:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/debug_exceptions.rb:28:in `call'",
      "lib/gitlab/middleware/path_traversal_check.rb:40:in `call'",
      "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:30:in `block (2 levels) in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:269:in `with_session_tracking'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:416:in `with_session_tracking'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:21:in `block in call'",
      "sentry-ruby (5.22.1) lib/sentry/hub.rb:59:in `with_scope'",
      "sentry-ruby (5.22.1) lib/sentry-ruby.rb:396:in `with_scope'",
      "sentry-ruby (5.22.1) lib/sentry/rack/capture_exceptions.rb:20:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/show_exceptions.rb:29:in `call'",
      "lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
      "lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'",
      "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `block in call'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `block in tagged'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:37:in `tagged'",
      "activesupport (7.0.8.7) lib/active_support/tagged_logging.rb:99:in `tagged'",
      "railties (7.0.8.7) lib/rails/rack/logger.rb:25:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/remote_ip.rb:93:in `call'",
      "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
      "lib/gitlab/middleware/request_context.rb:15:in `call'",
      "lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
      "request_store (1.5.1) lib/request_store/middleware.rb:19:in `call'",
      "rack (2.2.10) lib/rack/method_override.rb:24:in `call'",
      "rack (2.2.10) lib/rack/runtime.rb:22:in `call'",
      "rack-timeout (0.7.0) lib/rack/timeout/core.rb:154:in `block in call'",
      "rack-timeout (0.7.0) lib/rack/timeout/support/timeout.rb:19:in `timeout'",
      "rack-timeout (0.7.0) lib/rack/timeout/core.rb:153:in `call'",
      "config/initializers/fix_local_cache_middleware.rb:11:in `call'",
      "lib/gitlab/middleware/compressed_json.rb:44:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/executor.rb:14:in `call'",
      "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
      "rack (2.2.10) lib/rack/sendfile.rb:110:in `call'",
      "lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
      "lib/gitlab/metrics/requests_rack_middleware.rb:83:in `call'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:22:in `block in call'",
      "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
      "gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:21:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/request_id.rb:26:in `call'",
      "actionpack (7.0.8.7) lib/action_dispatch/middleware/host_authorization.rb:131:in `call'",
      "railties (7.0.8.7) lib/rails/engine.rb:530:in `call'",
      "railties (7.0.8.7) lib/rails/railtie.rb:226:in `public_send'",
      "railties (7.0.8.7) lib/rails/railtie.rb:226:in `method_missing'",
      "lib/gitlab/middleware/release_env.rb:12:in `call'",
      "rack (2.2.10) lib/rack/urlmap.rb:74:in `block in call'",
      "rack (2.2.10) lib/rack/urlmap.rb:58:in `each'",
      "rack (2.2.10) lib/rack/urlmap.rb:58:in `call'",
      "puma (6.5.0) lib/puma/configuration.rb:279:in `call'",
      "puma (6.5.0) lib/puma/request.rb:99:in `block in handle_request'",
      "puma (6.5.0) lib/puma/thread_pool.rb:389:in `with_force_shutdown'",
      "puma (6.5.0) lib/puma/request.rb:98:in `handle_request'",
      "puma (6.5.0) lib/puma/server.rb:468:in `process_client'",
      "puma (6.5.0) lib/puma/server.rb:249:in `block in run'",
      "puma (6.5.0) lib/puma/thread_pool.rb:166:in `block in spawn_thread'"
    ],
    "json.mem_objects": [
      14326594
    ],
    "json.shard": [
      "default"
    ],
    "json.correlation_id.keyword": [
      "a20324bd52c82b65a98249214d16b796"
    ],
    "json.correlation_id": [
      "a20324bd52c82b65a98249214d16b796"
    ],
    "json.db_ci_replica_cached_count": [
      0
    ],
    "json.db_ci_replica_wal_cached_count": [
      0
    ],
    "json.db_ci_count": [
      0
    ],
    "json.db_ci_txn_max_duration_s": [
      0
    ],
    "json.db_ci_replica_duration_s": [
      5.354
    ],
    "json.db_duration_s": [
      10.16631
    ],
    "json.action.keyword": [
      "execute"
    ],
    "json.format.keyword": [
      "*/*"
    ],
    "json.view_duration_s": [
      0
    ],
    "json.meta.client_id": [
      "user/xxxx"
    ],
    "json.redis_feature_flag_duration_s": [
      0.00032
    ],
    "json.type.keyword": [
      "api"
    ],
    "json.mem_mallocs": [
      4664855
    ],
    "json.redis_duration_s": [
      9.213776
    ],
    "json.params.key": [
      "operationName",
      "variables",
      "query",
      "graphql"
    ],
    "json.db_ci_txn_duration_s": [
      0
    ],
    "json.mem_total_bytes": [
      1181691968
    ],
    "type": [
      "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf"
    ],
    "json.mem_bytes": [
      608628208
    ],
    "json.redis_queues_catchall_a_read_bytes": [
      18274
    ],
    "publish_time": [
      "2025-02-05T18:57:39.412Z"
    ],
    "json.db_main_replica_txn_max_duration_s": [
      0
    ],
    "json.redis_feature_flag_calls": [
      1
    ],
    "json.graphql.variables.keyword": [
      "{}"
    ],
    "json.redis_queues_catchall_a_calls": [
      18274
    ],
    "json.db_ci_replica_txn_max_duration_s": [
      0
    ],
    "json.username": [
      "[redacted]"
    ],
    "json.redis_queues_metadata_read_bytes": [
      18278
    ],
    "json.db_ci_cached_count": [
      0
    ],
    "json.request_urgency": [
      "low"
    ],
    "json.queue_duration_s": [
      0.020119
    ],
    "json.redis_queues_catchall_a_duration_s": [
      4.686401
    ],
    "json.db_ci_txn_count": [
      0
    ],
    "json.username.keyword": [
      "[redacted]"
    ],
    "json.db_main_wal_cached_count": [
      0
    ],
    "json.db_main_txn_count": [
      0
    ],
    "json.environment.keyword": [
      "gprd"
    ],
    "json.redis_queues_metadata_duration_s": [
      4.525108
    ],
    "json.exception.message": [
      "Request ran for longer than 60000ms "
    ],
    "json.meta.user": [
      "[redacted]"
    ],
    "json.path.keyword": [
      "/api/graphql"
    ],
    "json.meta.user.keyword": [
      "[redacted]"
    ],
    "json.redis_read_bytes": [
      37342
    ],
    "json.action": [
      "execute"
    ],
    "json.redis_sessions_duration_s": [
      0.000856
    ],
    "json.meta.feature_category.keyword": [
      "not_owned"
    ],
    "json.redis_allowed_cross_slot_calls": [
      1
    ],
    "json.redis_calls": [
      27421
    ],
    "json.db_embedding_write_count": [
      0
    ],
    "json.meta.client_id.keyword": [
      "user/xxxx"
    ],
    "json.graphql.used_fields": [
      "SecurityPolicyProjectUnassignPayload.clientMutationId",
      "SecurityPolicyProjectUnassignPayload.errors",
      "Mutation.securityPolicyProjectUnassign"
    ],
    "json.redis_db_load_balancing_duration_s": [
      0.000826
    ],
    "json.db_ci_replica_wal_count": [
      9137
    ],
    "json.redis_sessions_write_bytes": [
      621
    ],
    "json.db_embedding_replica_duration_s": [
      0
    ],
    "json.db_main_replica_duration_s": [
      5.467
    ],
    "json.db_main_cached_count": [
      0
    ],
    "json.redis_rate_limiting_duration_s": [
      0.000265
    ],
    "json.cf_ipcountry": [
      "US"
    ],
    "json.redis_rate_limiting_write_bytes": [
      78
    ],
    "json.shard.keyword": [
      "default"
    ],
    "@timestamp": [
      "2025-02-05T18:57:39.540Z"
    ],
    "json.redis_feature_flag_write_bytes": [
      63
    ],
    "json.exception.class": [
      "Rack::Timeout::RequestTimeoutException"
    ],
    "json.db_ci_replica_txn_duration_s": [
      0
    ],
    "json.meta.remote_ip": [
      "xx.xx.xx.xx"
    ],
    "json.method.keyword": [
      "POST"
    ],
    "json.meta.http_router_rule_action.keyword": [
      "proxy"
    ],
    "json.db_main_replica_wal_count": [
      9138
    ],
    "json.graphql.operation_name": [
      "securityPolicyProjectUnassign"
    ],
    "json.logtag": [
      "F"
    ],
    "json.db_main_write_count": [
      0
    ],
    "json.db_main_replica_wal_cached_count": [
      0
    ],
    "json.db_embedding_replica_txn_count": [
      0
    ],
    "json.redis_queues_metadata_calls": [
      9139
    ],
    "host.name": [
      "pubsubbeat-pubsub-rails-inf-gprd-6bd86f44c5-np4wf"
    ],
    "json.component.keyword": [
      "gitlab"
    ],
    "json.tag": [
      "api-rails.var.log.containers.gitlab-webservice-api-59b589c4db-bhd29_gitlab_webservice-db794b5244717e92d85d0091d3604d6bf084558831c96f5dc0e5d05edcada1ca.log"
    ],
    "json.cf_ray.keyword": [
      "90d51ea35317a384-ATL"
    ],
    "kubernetes.container_name": [
      "webservice"
    ],
    "json.path": [
      "/api/graphql"
    ],
    "kubernetes.pod_ip.keyword": [
      "10.64.99.37"
    ],
    "json.pid": [
      75
    ],
    "json.stage": [
      "main"
    ],
    "json.db_main_replica_count": [
      9159
    ],
    "json.graphql.used_fields.keyword": [
      "SecurityPolicyProjectUnassignPayload.clientMutationId",
      "SecurityPolicyProjectUnassignPayload.errors",
      "Mutation.securityPolicyProjectUnassign"
    ],
    "kubernetes.zone.keyword": [
      "us-east1-b"
    ],
    "json.request_urgency.keyword": [
      "low"
    ],
    "json.redis_write_bytes": [
      14876371
    ],
    "kubernetes.zone": [
      "us-east1-b"
    ],
    "json.component": [
      "gitlab"
    ],
    "json.cpu_s": [
      37.570698
    ],
    "json.tag.keyword": [
      "api-rails.var.log.containers.gitlab-webservice-api-59b589c4db-bhd29_gitlab_webservice-db794b5244717e92d85d0091d3604d6bf084558831c96f5dc0e5d05edcada1ca.log"
    ],
    "json.redis_db_load_balancing_calls": [
      3
    ],
    "hour": [
      18
    ],
    "json.controller": [
      "GraphqlController"
    ],
    "json.subcomponent": [
      "production_json"
    ],
    "json.db_embedding_replica_write_count": [
      0
    ],
    "json.redis_sessions_read_bytes": [
      552
    ],
    "json.db_main_duration_s": [
      0.001
    ],
    "json.db_main_replica_cached_count": [
      0
    ],
    "kubernetes.host": [
      "gke-gprd-us-east1-b-generic-3-bc3a5739-bd99"
    ],
    "json.cf_ipcountry.keyword": [
      "US"
    ],
    "json.stage.keyword": [
      "main"
    ],
    "json.db_main_txn_duration_s": [
      0
    ],
    "json.method": [
      "POST"
    ],
    "json.db_embedding_wal_cached_count": [
      0
    ],
    "kubernetes.region.keyword": [
      "us-east1"
    ],
    "json.meta.remote_ip.keyword": [
      "xx.xx.xx.xx"
    ],
    "json.tier": [
      "sv"
    ],
    "ua_string_base": [
      "Mozilla"
    ],
    "kubernetes.namespace_name": [
      "gitlab"
    ],
    "json.db_ci_wal_count": [
      0
    ],
    "json.remote_ip.keyword": [
      "xx.xx.xx.xx"
    ],
    "json.status": [
      500
    ],
    "json.db_embedding_replica_wal_cached_count": [
      0
    ],
    "json.meta.user_id": [
      4240729
    ],
    "json.controller.keyword": [
      "GraphqlController"
    ],
    "json.db_embedding_txn_count": [
      0
    ]
  }
}

Output of checks

Results of GitLab environment info

Expand for output related to GitLab environment info

(For installations with omnibus-gitlab package run and paste the output of: \\\`sudo gitlab-rake gitlab:env:info\\\`) (For installations from source run and paste the output of: \\\`sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production\\\`)

Results of GitLab application Check

Expand for output related to the GitLab application check

(For installations with omnibus-gitlab package run and paste the output of: \`sudo gitlab-rake gitlab:check SANITIZE=true\`) (For installations from source run and paste the output of: \`sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production SANITIZE=true\`) (we will only investigate if the tests are passing)

Possible fixes