Database statement timeout in /api/v4/internal/allowed
https://sentry.gitlab.net/gitlab/gitlabcom/issues/1171842/
PG::QueryCanceled: ERROR: canceling statement due to statement timeout
active_record/connection_adapters/postgresql_adapter.rb:611:in `exec_params'
@connection.exec_params(sql, type_casted_binds)
active_record/connection_adapters/postgresql_adapter.rb:611:in `block (2 levels) in exec_no_cache'
@connection.exec_params(sql, type_casted_binds)
active_support/dependencies/interlock.rb:48:in `block in permit_concurrent_loads'
yield
active_support/concurrency/share_lock.rb:187:in `yield_shares'
yield
active_support/dependencies/interlock.rb:47:in `permit_concurrent_loads'
@lock.yield_shares(compatible: [:load]) do
...
(167 additional frame(s) were not displayed)
ActiveRecord::QueryCanceled: PG::QueryCanceled: ERROR: canceling statement due to statement timeout
: SELECT "namespaces"."id" AS t0_r0, "namespaces"."name" AS t0_r1, "namespaces"."path" AS t0_r2, "namespaces"."owner_id" AS t0_r3, "namespaces"."created_at" AS t0_r4, "namespaces"."updated_at" AS t0_r5, "namespaces"."type" AS t0_r6, "namespaces"."description" AS t0_r7, "namespaces"."avatar" AS t0_r8, "namespaces"."membership_lock" AS t0_r9, "namespaces"."share_with_group_lock" AS t0_r10, "namespaces"."visibility_level" AS t0_r11, "namespaces"."request_access_enabled" AS t0_r12, "namespaces"."ldap_sync_status" AS t0_r13, "namespaces"."ldap_sync_error" AS t0_r14, "namespaces"."ldap_sync_last_update_at" AS t0_r15, "namespaces"."ldap_sync_last_successful_update_at" AS t0_r16, "namespaces"."ldap_sync_last_sync_at" AS t0_r17, "namespaces"."lfs_enabled" AS t0_r18, "namespaces"."description_html" AS t0_r19, "namespaces"."parent_id" AS t0_r20, "namespaces"."shared_runners_minutes_limit" AS t0_r21, "namespaces"."repository_size_limit" AS t0_r22, "namespaces"."require_two_factor_authentication" AS t0_r23, "namespaces"."two_factor_grace_period" AS t0_r24, "namespaces"."cached_markdown_version" AS t0_r25, "namespaces"."plan_id" AS t0_r26, "namespaces"."project_creation_level" AS t0_r27, "namespaces"."runners_token" AS t0_r28, "namespaces"."trial_ends_on" AS t0_r29, "namespaces"."file_template_project_id" AS t0_r30, "namespaces"."saml_discovery_token" AS t0_r31, "namespaces"."runners_token_encrypted" AS t0_r32, "namespaces"."custom_project_templates_group_id" AS t0_r33, "namespaces"."auto_devops_enabled" AS t0_r34, "namespaces"."extra_shared_runners_minutes_limit" AS t0_r35, "namespaces"."last_ci_minutes_notification_at" AS t0_r36, "namespaces"."last_ci_minutes_usage_notification_level" AS t0_r37, "namespaces"."subgroup_creation_level" AS t0_r38, "namespaces"."emails_disabled" AS t0_r39, "namespaces"."max_pages_size" AS t0_r40, "namespaces"."max_artifacts_size" AS t0_r41, "namespaces"."mentions_disabled" AS t0_r42, "routes"."id" AS t1_r0, "routes"."source_id" AS t1_r1, "routes"."source_type" AS t1_r2, "routes"."path" AS t1_r3, "routes"."created_at" AS t1_r4, "routes"."updated_at" AS t1_r5, "routes"."name" AS t1_r6 FROM "namespaces" LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id" AND "routes"."source_type" = 'Namespace' WHERE "routes"."path" IS NULL LIMIT 1
ActiveRecord::QueryCanceled: PG::QueryCanceled: ERROR: canceling statement due to statement timeout
: SELECT "namespaces"."id" AS t0_r0, "namespaces"."name" AS t0_r1, "namespaces"."path" AS t0_r2, "namespaces"."owner_id" AS t0_r3, "namespaces"."created_at" AS t0_r4, "namespaces"."updated_at" AS t0_r5, "namespaces"."type" AS t0_r6, "namespaces"."description" AS t0_r7, "namespaces"."avatar" AS t0_r8, "namespaces"."membership_lock" AS t0_r9, "namespaces"."share_with_group_lock" AS t0_r10, "namespaces"."visibility_level" AS t0_r11, "namespaces"."request_access_enabled" AS t0_r12, "namespaces"."ldap_sync_status" AS t0_r13, "namespaces"."ldap_sync_error" AS t0_r14, "namespaces"."ldap_sync_last_update_at" AS t0_r15, "namespaces"."ldap_sync_last_successful_update_at" AS t0_r16, "namespaces"."ldap_sync_last_sync_at" AS t0_r17, "namespaces"."lfs_enabled" AS t0_r18, "namespaces"."description_html" AS t0_r19, "namespaces"."parent_id" AS t0_r20, "namespaces"."shared_runners_minutes_limit" AS t0_r21, "namespaces"."repository_size_limit" AS t0_r22, "namespaces"."require_two_factor_authentication" AS t0_r23, "namespaces"."two_factor_grace_period" AS t0_r24, "namespaces"."cached_markdown_version" AS t0_r25, "namespaces"."plan_id" AS t0_r26, "namespaces"."project_creation_level" AS t0_r27, "namespaces"."runners_token" AS t0_r28, "namespaces"."trial_ends_on" AS t0_r29, "namespaces"."file_template_project_id" AS t0_r30, "namespaces"."saml_discovery_token" AS t0_r31, "namespaces"."runners_token_encrypted" AS t0_r32, "namespaces"."custom_project_templates_group_id" AS t0_r33, "namespaces"."auto_devops_enabled" AS t0_r34, "namespaces"."extra_shared_runners_minutes_limit" AS t0_r35, "namespaces"."last_ci_minutes_notification_at" AS t0_r36, "namespaces"."last_ci_minutes_usage_notification_level" AS t0_r37, "namespaces"."subgroup_creation_level" AS t0_r38, "namespaces"."emails_disabled" AS t0_r39, "namespaces"."max_pages_size" AS t0_r40, "namespaces"."max_artifacts_size" AS t0_r41, "namespaces"."mentions_disabled" AS t0_r42, "routes"."id" AS t1_r0, "routes"."source_id" AS t1_r1, "routes"."source_type" AS t1_r2, "routes"."path" AS t1_r3, "routes"."created_at" AS t1_r4, "routes"."updated_at" AS t1_r5, "routes"."name" AS t1_r6 FROM "namespaces" LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id" AND "routes"."source_type" = 'Namespace' WHERE "routes"."path" IS NULL LIMIT 1SQL:
SELECT
"namespaces"."id" AS t0_r0,
"namespaces"."name" AS t0_r1,
"namespaces"."path" AS t0_r2,
"namespaces"."owner_id" AS t0_r3,
"namespaces"."created_at" AS t0_r4,
"namespaces"."updated_at" AS t0_r5,
"namespaces"."type" AS t0_r6,
"namespaces"."description" AS t0_r7,
"namespaces"."avatar" AS t0_r8,
"namespaces"."membership_lock" AS t0_r9,
"namespaces"."share_with_group_lock" AS t0_r10,
"namespaces"."visibility_level" AS t0_r11,
"namespaces"."request_access_enabled" AS t0_r12,
"namespaces"."ldap_sync_status" AS t0_r13,
"namespaces"."ldap_sync_error" AS t0_r14,
"namespaces"."ldap_sync_last_update_at" AS t0_r15,
"namespaces"."ldap_sync_last_successful_update_at" AS t0_r16,
"namespaces"."ldap_sync_last_sync_at" AS t0_r17,
"namespaces"."lfs_enabled" AS t0_r18,
"namespaces"."description_html" AS t0_r19,
"namespaces"."parent_id" AS t0_r20,
"namespaces"."shared_runners_minutes_limit" AS t0_r21,
"namespaces"."repository_size_limit" AS t0_r22,
"namespaces"."require_two_factor_authentication" AS t0_r23,
"namespaces"."two_factor_grace_period" AS t0_r24,
"namespaces"."cached_markdown_version" AS t0_r25,
"namespaces"."plan_id" AS t0_r26,
"namespaces"."project_creation_level" AS t0_r27,
"namespaces"."runners_token" AS t0_r28,
"namespaces"."trial_ends_on" AS t0_r29,
"namespaces"."file_template_project_id" AS t0_r30,
"namespaces"."saml_discovery_token" AS t0_r31,
"namespaces"."runners_token_encrypted" AS t0_r32,
"namespaces"."custom_project_templates_group_id" AS t0_r33,
"namespaces"."auto_devops_enabled" AS t0_r34,
"namespaces"."extra_shared_runners_minutes_limit" AS t0_r35,
"namespaces"."last_ci_minutes_notification_at" AS t0_r36,
"namespaces"."last_ci_minutes_usage_notification_level" AS t0_r37,
"namespaces"."subgroup_creation_level" AS t0_r38,
"namespaces"."emails_disabled" AS t0_r39,
"namespaces"."max_pages_size" AS t0_r40,
"namespaces"."max_artifacts_size" AS t0_r41,
"namespaces"."mentions_disabled" AS t0_r42,
"routes"."id" AS t1_r0,
"routes"."source_id" AS t1_r1,
"routes"."source_type" AS t1_r2,
"routes"."path" AS t1_r3,
"routes"."created_at" AS t1_r4,
"routes"."updated_at" AS t1_r5,
"routes"."name" AS t1_r6
FROM
"namespaces"
LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id"
AND "routes"."source_type" = 'Namespace'
WHERE
"routes"."path" IS NULL
LIMIT
1EXPLAIN output (https://explain.depesz.com/s/tsMY):
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=1779063.55..2195873.41 rows=1 width=401) (actual time=14891.437..14891.442 rows=1 loops=1)
Output: namespaces.id, namespaces.name, namespaces.path, namespaces.owner_id, namespaces.created_at, namespaces.updated_at, namespaces.type, namespaces.description, namespaces.avatar, namespaces.membership_lock, namespaces.share_with_group_lock, namespaces.visibility_level, namespaces.request_access_enabled, namespaces.ldap_sync_status, namespaces.ldap_sync_error, namespaces.ldap_sync_last_update_at, namespaces.ldap_sync_last_successful_update_at, namespaces.ldap_sync_last_sync_at, namespaces.lfs_enabled, namespaces.description_html, namespaces.parent_id, namespaces.shared_runners_minutes_limit, namespaces.repository_size_limit, namespaces.require_two_factor_authentication, namespaces.two_factor_grace_period, namespaces.cached_markdown_version, namespaces.plan_id, namespaces.project_creation_level, namespaces.runners_token, namespaces.trial_ends_on, namespaces.file_template_project_id, namespaces.saml_discovery_token, namespaces.runners_token_encrypted, namespaces.custom_project_templates_group_id, namespaces.auto_devops_enabled, namespaces.extra_shared_runners_minutes_limit, namespaces.last_ci_minutes_notification_at, namespaces.last_ci_minutes_usage_notification_level, namespaces.subgroup_creation_level, namespaces.emails_disabled, namespaces.max_pages_size, namespaces.max_artifacts_size, namespaces.mentions_disabled, routes.id, routes.source_id, routes.source_type, routes.path, routes.created_at, routes.updated_at, routes.name
-> Merge Right Join (cost=1779063.55..2195873.41 rows=1 width=401) (actual time=14891.434..14891.434 rows=1 loops=1)
Output: namespaces.id, namespaces.name, namespaces.path, namespaces.owner_id, namespaces.created_at, namespaces.updated_at, namespaces.type, namespaces.description, namespaces.avatar, namespaces.membership_lock, namespaces.share_with_group_lock, namespaces.visibility_level, namespaces.request_access_enabled, namespaces.ldap_sync_status, namespaces.ldap_sync_error, namespaces.ldap_sync_last_update_at, namespaces.ldap_sync_last_successful_update_at, namespaces.ldap_sync_last_sync_at, namespaces.lfs_enabled, namespaces.description_html, namespaces.parent_id, namespaces.shared_runners_minutes_limit, namespaces.repository_size_limit, namespaces.require_two_factor_authentication, namespaces.two_factor_grace_period, namespaces.cached_markdown_version, namespaces.plan_id, namespaces.project_creation_level, namespaces.runners_token, namespaces.trial_ends_on, namespaces.file_template_project_id, namespaces.saml_discovery_token, namespaces.runners_token_encrypted, namespaces.custom_project_templates_group_id, namespaces.auto_devops_enabled, namespaces.extra_shared_runners_minutes_limit, namespaces.last_ci_minutes_notification_at, namespaces.last_ci_minutes_usage_notification_level, namespaces.subgroup_creation_level, namespaces.emails_disabled, namespaces.max_pages_size, namespaces.max_artifacts_size, namespaces.mentions_disabled, routes.id, routes.source_id, routes.source_type, routes.path, routes.created_at, routes.updated_at, routes.name
Merge Cond: (routes.source_id = namespaces.id)
Filter: (routes.path IS NULL)
Rows Removed by Filter: 3050647
-> Sort (cost=1779060.02..1794630.85 rows=6228332 width=78) (actual time=8794.694..9702.792 rows=3050648 loops=1)
Output: routes.id, routes.source_id, routes.source_type, routes.path, routes.created_at, routes.updated_at, routes.name
Sort Key: routes.source_id
Sort Method: external merge Disk: 442208kB
-> Index Scan using index_routes_on_source_type_and_source_id on public.routes (cost=0.56..542450.60 rows=6228332 width=78) (actual time=0.020..6421.947 rows=6323605 loops=1)
Output: routes.id, routes.source_id, routes.source_type, routes.path, routes.created_at, routes.updated_at, routes.name
Index Cond: ((routes.source_type)::text = 'Namespace'::text)
-> Index Scan using namespaces_pkey on public.namespaces (cost=0.43..350298.27 rows=6321117 width=323) (actual time=0.034..3502.448 rows=3050648 loops=1)
Output: namespaces.id, namespaces.name, namespaces.path, namespaces.owner_id, namespaces.created_at, namespaces.updated_at, namespaces.type, namespaces.description, namespaces.avatar, namespaces.membership_lock, namespaces.share_with_group_lock, namespaces.visibility_level, namespaces.request_access_enabled, namespaces.ldap_sync_status, namespaces.ldap_sync_error, namespaces.ldap_sync_last_update_at, namespaces.ldap_sync_last_successful_update_at, namespaces.ldap_sync_last_sync_at, namespaces.lfs_enabled, namespaces.description_html, namespaces.parent_id, namespaces.shared_runners_minutes_limit, namespaces.repository_size_limit, namespaces.require_two_factor_authentication, namespaces.two_factor_grace_period, namespaces.cached_markdown_version, namespaces.plan_id, namespaces.project_creation_level, namespaces.runners_token, namespaces.trial_ends_on, namespaces.file_template_project_id, namespaces.saml_discovery_token, namespaces.runners_token_encrypted, namespaces.custom_project_templates_group_id, namespaces.auto_devops_enabled, namespaces.extra_shared_runners_minutes_limit, namespaces.last_ci_minutes_notification_at, namespaces.last_ci_minutes_usage_notification_level, namespaces.subgroup_creation_level, namespaces.emails_disabled, namespaces.max_pages_size, namespaces.max_artifacts_size, namespaces.mentions_disabled
Planning time: 0.469 ms
Execution time: 14987.201 ms
(18 rows)Edited by Stan Hu