Failures in E2E tests due to N+1 in groups API
https://gitlab.com/gitlab-org/quality/nightly/-/jobs/351507350
nplus1-instance-image-ee 3/5 job
test failure stacktraces
Failures:
7171 1) Create Commit data user views raw email patch
7172 Failure/Error:
7173 project_push = Resource::Repository::ProjectPush.fabricate! do |push|
7174 push.file_name = 'README.md'
7175 push.file_content = '# This is a test project'
7176 push.commit_message = 'Add README.md'
7177 end
7178
7179 QA::Resource::ApiFabricator::ResourceFabricationFailedError:
7180 Fabrication of QA::Resource::Group using the API failed (400) with `{"message":"Failed to save group {:path=\u003e[\"has already been taken\"]}"}`.
7181 # ./qa/resource/api_fabricator.rb:81:in `api_post'
7182 # ./qa/resource/api_fabricator.rb:31:in `fabricate_via_api!'
7183 # ./qa/resource/group.rb:50:in `rescue in fabricate_via_api!'
7184 # ./qa/resource/group.rb:47:in `fabricate_via_api!'
7185 # ./qa/resource/base.rb:46:in `block (2 levels) in fabricate_via_api!'
7186 # ./qa/resource/base.rb:130:in `log_fabrication'
7187 # ./qa/resource/base.rb:46:in `block in fabricate_via_api!'
7188 # ./qa/resource/base.rb:113:in `do_fabricate!'
7189 # ./qa/resource/base.rb:45:in `fabricate_via_api!'
7190 # ./qa/resource/base.rb:19:in `fabricate!'
7191 # ./qa/resource/project.rb:23:in `block in <class:Project>'
7192 # ./qa/resource/base.rb:103:in `instance_exec'
7193 # ./qa/resource/base.rb:103:in `attribute_value'
7194 # ./qa/resource/base.rb:89:in `populate_attribute'
7195 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7196 # ./qa/resource/project.rb:27:in `block in <class:Project>'
7197 # ./qa/resource/base.rb:103:in `instance_exec'
7198 # ./qa/resource/base.rb:103:in `attribute_value'
7199 # ./qa/resource/base.rb:89:in `populate_attribute'
7200 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7201 # ./qa/resource/project.rb:82:in `api_get_path'
7202 # ./qa/resource/api_fabricator.rb:61:in `api_get'
7203 # ./qa/resource/project.rb:76:in `fabricate_via_api!'
7204 # ./qa/resource/base.rb:46:in `block (2 levels) in fabricate_via_api!'
7205 # ./qa/resource/base.rb:130:in `log_fabrication'
7206 # ./qa/resource/base.rb:46:in `block in fabricate_via_api!'
7207 # ./qa/resource/base.rb:113:in `do_fabricate!'
7208 # ./qa/resource/base.rb:45:in `fabricate_via_api!'
7209 # ./qa/resource/base.rb:19:in `fabricate!'
7210 # ./qa/resource/repository/project_push.rb:11:in `block in <class:ProjectPush>'
7211 # ./qa/resource/base.rb:103:in `instance_exec'
7212 # ./qa/resource/base.rb:103:in `attribute_value'
7213 # ./qa/resource/base.rb:89:in `populate_attribute'
7214 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7215 # ./qa/resource/repository/project_push.rb:28:in `repository_http_uri'
7216 # ./qa/resource/repository/push.rb:56:in `block in fabricate!'
7217 # ./qa/scenario/actable.rb:16:in `perform'
7218 # ./qa/git/repository.rb:33:in `block (2 levels) in perform'
7219 # ./qa/git/repository.rb:33:in `chdir'
7220 # ./qa/git/repository.rb:33:in `block in perform'
7221 # ./qa/git/repository.rb:32:in `perform'
7222 # ./qa/resource/repository/push.rb:49:in `fabricate!'
7223 # ./qa/resource/repository/project_push.rb:36:in `fabricate!'
7224 # ./qa/resource/base.rb:30:in `block (2 levels) in fabricate_via_browser_ui!'
7225 # ./qa/resource/base.rb:130:in `log_fabrication'
7226 # ./qa/resource/base.rb:30:in `block in fabricate_via_browser_ui!'
7227 # ./qa/resource/base.rb:113:in `do_fabricate!'
7228 # ./qa/resource/base.rb:29:in `fabricate_via_browser_ui!'
7229 # ./qa/resource/base.rb:21:in `rescue in fabricate!'
7230 # ./qa/resource/base.rb:18:in `fabricate!'
7231 # ./qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb:12:in `block (3 levels) in <module:QA>'
7232 # ./qa/specs/runner.rb:69:in `perform'
7233 # ./qa/scenario/template.rb:10:in `block in perform'
7234 # ./qa/scenario/template.rb:8:in `tap'
7235 # ./qa/scenario/template.rb:8:in `perform'
7236 # ./qa/scenario/template.rb:33:in `perform'
7237 # ./qa/scenario/template.rb:10:in `block in perform'
7238 # ./qa/scenario/template.rb:8:in `tap'
7239 # ./qa/scenario/template.rb:8:in `perform'
7240 # ./qa/scenario/bootable.rb:28:in `launch!'
7241 # ------------------
7242 # --- Caused by: ---
7243 # NotImplementedError:
7244 # NotImplementedError
7245 # ./qa/resource/base.rb:41:in `fabricate_via_api!'
7246 2) Create Commit data user views raw commit diff
7247 Failure/Error:
7248 project_push = Resource::Repository::ProjectPush.fabricate! do |push|
7249 push.file_name = 'README.md'
7250 push.file_content = '# This is a test project'
7251 push.commit_message = 'Add README.md'
7252 end
7253
7254 QA::Resource::ApiFabricator::ResourceFabricationFailedError:
7255 Fabrication of QA::Resource::Group using the API failed (400) with `{"message":"Failed to save group {:path=\u003e[\"has already been taken\"]}"}`.
7256 # ./qa/resource/api_fabricator.rb:81:in `api_post'
7257 # ./qa/resource/api_fabricator.rb:31:in `fabricate_via_api!'
7258 # ./qa/resource/group.rb:50:in `rescue in fabricate_via_api!'
7259 # ./qa/resource/group.rb:47:in `fabricate_via_api!'
7260 # ./qa/resource/base.rb:46:in `block (2 levels) in fabricate_via_api!'
7261 # ./qa/resource/base.rb:130:in `log_fabrication'
7262 # ./qa/resource/base.rb:46:in `block in fabricate_via_api!'
7263 # ./qa/resource/base.rb:113:in `do_fabricate!'
7264 # ./qa/resource/base.rb:45:in `fabricate_via_api!'
7265 # ./qa/resource/base.rb:19:in `fabricate!'
7266 # ./qa/resource/project.rb:23:in `block in <class:Project>'
7267 # ./qa/resource/base.rb:103:in `instance_exec'
7268 # ./qa/resource/base.rb:103:in `attribute_value'
7269 # ./qa/resource/base.rb:89:in `populate_attribute'
7270 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7271 # ./qa/resource/project.rb:27:in `block in <class:Project>'
7272 # ./qa/resource/base.rb:103:in `instance_exec'
7273 # ./qa/resource/base.rb:103:in `attribute_value'
7274 # ./qa/resource/base.rb:89:in `populate_attribute'
7275 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7276 # ./qa/resource/project.rb:82:in `api_get_path'
7277 # ./qa/resource/api_fabricator.rb:61:in `api_get'
7278 # ./qa/resource/project.rb:76:in `fabricate_via_api!'
7279 # ./qa/resource/base.rb:46:in `block (2 levels) in fabricate_via_api!'
7280 # ./qa/resource/base.rb:130:in `log_fabrication'
7281 # ./qa/resource/base.rb:46:in `block in fabricate_via_api!'
7282 # ./qa/resource/base.rb:113:in `do_fabricate!'
7283 # ./qa/resource/base.rb:45:in `fabricate_via_api!'
7284 # ./qa/resource/base.rb:19:in `fabricate!'
7285 # ./qa/resource/repository/project_push.rb:11:in `block in <class:ProjectPush>'
7286 # ./qa/resource/base.rb:103:in `instance_exec'
7287 # ./qa/resource/base.rb:103:in `attribute_value'
7288 # ./qa/resource/base.rb:89:in `populate_attribute'
7289 # ./qa/resource/base.rb:156:in `block (2 levels) in attribute'
7290 # ./qa/resource/repository/project_push.rb:28:in `repository_http_uri'
7291 # ./qa/resource/repository/push.rb:56:in `block in fabricate!'
7292 # ./qa/scenario/actable.rb:16:in `perform'
7293 # ./qa/git/repository.rb:33:in `block (2 levels) in perform'
7294 # ./qa/git/repository.rb:33:in `chdir'
7295 # ./qa/git/repository.rb:33:in `block in perform'
7296 # ./qa/git/repository.rb:32:in `perform'
7297 # ./qa/resource/repository/push.rb:49:in `fabricate!'
7298 # ./qa/resource/repository/project_push.rb:36:in `fabricate!'
7299 # ./qa/resource/base.rb:30:in `block (2 levels) in fabricate_via_browser_ui!'
7300 # ./qa/resource/base.rb:130:in `log_fabrication'
7301 # ./qa/resource/base.rb:30:in `block in fabricate_via_browser_ui!'
7302 # ./qa/resource/base.rb:113:in `do_fabricate!'
7303 # ./qa/resource/base.rb:29:in `fabricate_via_browser_ui!'
7304 # ./qa/resource/base.rb:21:in `rescue in fabricate!'
7305 # ./qa/resource/base.rb:18:in `fabricate!'
7306 # ./qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb:12:in `block (3 levels) in <module:QA>'
7307 # ./qa/specs/runner.rb:69:in `perform'
7308 # ./qa/scenario/template.rb:10:in `block in perform'
7309 # ./qa/scenario/template.rb:8:in `tap'
7310 # ./qa/scenario/template.rb:8:in `perform'
7311 # ./qa/scenario/template.rb:33:in `perform'
7312 # ./qa/scenario/template.rb:10:in `block in perform'
7313 # ./qa/scenario/template.rb:8:in `tap'
7314 # ./qa/scenario/template.rb:8:in `perform'
7315 # ./qa/scenario/bootable.rb:28:in `launch!'
7316 # ------------------
7317 # --- Caused by: ---
7318 # NotImplementedError:
7319 # NotImplementedError
7320 # ./qa/resource/base.rb:41:in `fabricate_via_api!'
N+1 when using the groups API. It was expected to be fixed by !20023 (merged) but maybe it's a different problem with the same API?
{
"time": "2019-11-15T04:56:59.604Z",
"severity": "INFO",
"duration": 1719.03,
"db": 120.49,
"view": 1598.54,
"status": 500,
"method": "GET",
"path": "/api/v4/groups/gitlab-qa-sandbox-group%2Fqa-test-2019-11-15-04-42-53-7ecd6f2c9e03498e",
"params": [
{
"key": "private_token",
"value": "[FILTERED]"
}
],
"host": "gitlab-ee-a9fde828.test",
"remote_ip": "172.19.0.3, 127.0.0.1",
"ua": "rest-client/2.0.2 (linux-gnu x86_64) ruby/2.6.5p114",
"route": "/api/:version/groups/:id",
"user_id": 1,
"username": "root",
"exception": {
"class": "Gitlab::GitalyClient::TooManyInvocationsError",
"message": "GitalyClient#call called 31 times from single request. Potential n+1?\nThe following call site called into Gitaly 15 times:\nlib/gitlab/gitaly_client.rb:163:in `execute'\nlib/gitlab/gitaly_client.rb:147:in `block in call'\nlib/gitlab/gitaly_client.rb:174:in `measure_timings'\nlib/gitlab/gitaly_client.rb:146:in `call'\nlib/gitlab/gitaly_client/ref_service.rb:42:in `default_branch_name'\nlib/gitlab/git/repository.rb:90:in `root_ref'\napp/models/repository.rb:524:in `root_ref'\nlib/gitlab/repository_cache_adapter.rb:84:in `block (2 levels) in cache_method_asymmetrically'\nlib/gitlab/repository_cache.rb:43:in `fetch_without_caching_false'\nlib/gitlab/repository_cache_adapter.rb:174:in `block (2 levels) in cache_method_output_asymmetrically'\nlib/gitlab/safe_request_store.rb:12:in `fetch'\nlib/gitlab/repository_cache.rb:24:in `fetch'\nlib/gitlab/repository_cache_adapter.rb:173:in `block in cache_method_output_asymmetrically'\nlib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'\nlib/gitlab/repository_cache_adapter.rb:187:in `block in memoize_method_output'\nlib/gitlab/repository_cache_adapter.rb:196:in `no_repository_fallback'\nlib/gitlab/repository_cache_adapter.rb:186:in `memoize_method_output'\nlib/gitlab/repository_cache_adapter.rb:172:in `cache_method_output_asymmetrically'\nlib/gitlab/repository_cache_adapter.rb:83:in `block in cache_method_asymmetrically'\napp/models/project.rb:1490:in `default_branch'\nee/lib/gitlab/ip_address_state.rb:10:in `with'\nlib/api/api_guard.rb:164:in `call'\nee/lib/gitlab/jira/middleware.rb:19:in `call'\\nlib/gitlab/gitaly_client.rb:163:in `execute'\nlib/gitlab/gitaly_client.rb:147:in `block in call'\nlib/gitlab/gitaly_client.rb:174:in `measure_timings'\nlib/gitlab/gitaly_client.rb:146:in `call'\nlib/gitlab/gitaly_client/repository_service.rb:99:in `has_local_branches?'\nlib/gitlab/git/repository.rb:1061:in `block in uncached_has_local_branches?'\nlib/gitlab/git/wraps_gitaly_errors.rb:7:in `wrapped_gitaly_errors'\nlib/gitlab/git/repository.rb:1060:in `uncached_has_local_branches?'\nlib/gitlab/git/repository.rb:161:in `block in has_local_branches?'\nlib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'\nlib/gitlab/git/repository.rb:160:in `has_local_branches?'\napp/models/repository.rb:568:in `has_visible_content?'\nlib/gitlab/repository_cache_adapter.rb:84:in `block (2 levels) in cache_method_asymmetrically'\nlib/gitlab/repository_cache.rb:43:in `fetch_without_caching_false'\nlib/gitlab/repository_cache_adapter.rb:174:in `block (2 levels) in cache_method_output_asymmetrically'\nlib/gitlab/safe_request_store.rb:12:in `fetch'\nlib/gitlab/repository_cache.rb:24:in `fetch'\nlib/gitlab/repository_cache_adapter.rb:173:in `block in cache_method_output_asymmetrically'\nlib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'\nlib/gitlab/repository_cache_adapter.rb:187:in `block in memoize_method_output'\nlib/gitlab/repository_cache_adapter.rb:196:in `no_repository_fallback'\nlib/gitlab/repository_cache_adapter.rb:186:in `memoize_method_output'\nlib/gitlab/repository_cache_adapter.rb:172:in `cache_method_output_asymmetrically'\nlib/gitlab/repository_cache_adapter.rb:83:in `block in cache_method_asymmetrically'\napp/models/repository.rb:542:in `empty?'\napp/models/project.rb:727:in `empty_repo?'\nee/lib/gitlab/ip_address_state.rb:10:in `with'\nlib/api/api_guard.rb:164:in `call'\nee/lib/gitlab/jira/middleware.rb:19:in `call'\n",
"backtrace": [
"lib/gitlab/gitaly_client.rb:277:in `enforce_gitaly_request_limits'",
"lib/gitlab/gitaly_client.rb:163:in `execute'",
"lib/gitlab/gitaly_client.rb:147:in `block in call'",
"lib/gitlab/gitaly_client.rb:174:in `measure_timings'",
"lib/gitlab/gitaly_client.rb:146:in `call'",
"lib/gitlab/gitaly_client/repository_service.rb:99:in `has_local_branches?'",
"lib/gitlab/git/repository.rb:1061:in `block in uncached_has_local_branches?'",
"lib/gitlab/git/wraps_gitaly_errors.rb:7:in `wrapped_gitaly_errors'",
"lib/gitlab/git/repository.rb:1060:in `uncached_has_local_branches?'",
"lib/gitlab/git/repository.rb:161:in `block in has_local_branches?'",
"lib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'",
"lib/gitlab/git/repository.rb:160:in `has_local_branches?'",
"app/models/repository.rb:568:in `has_visible_content?'",
"lib/gitlab/repository_cache_adapter.rb:84:in `block (2 levels) in cache_method_asymmetrically'",
"lib/gitlab/repository_cache.rb:43:in `fetch_without_caching_false'",
"lib/gitlab/repository_cache_adapter.rb:174:in `block (2 levels) in cache_method_output_asymmetrically'",
"lib/gitlab/safe_request_store.rb:12:in `fetch'",
"lib/gitlab/repository_cache.rb:24:in `fetch'",
"lib/gitlab/repository_cache_adapter.rb:173:in `block in cache_method_output_asymmetrically'",
"lib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'",
"lib/gitlab/repository_cache_adapter.rb:187:in `block in memoize_method_output'",
"lib/gitlab/repository_cache_adapter.rb:196:in `no_repository_fallback'",
"lib/gitlab/repository_cache_adapter.rb:186:in `memoize_method_output'",
"lib/gitlab/repository_cache_adapter.rb:172:in `cache_method_output_asymmetrically'",
"lib/gitlab/repository_cache_adapter.rb:83:in `block in cache_method_asymmetrically'",
"app/models/repository.rb:542:in `empty?'",
"app/models/project.rb:727:in `empty_repo?'",
"ee/lib/gitlab/ip_address_state.rb:10:in `with'",
"lib/api/api_guard.rb:164:in `call'",
"ee/lib/gitlab/jira/middleware.rb:19:in `call'"
]
},
"queue_duration": 87.39,
"gitaly_calls": 31,
"gitaly_duration": 288.37,
"rugged_calls": 3,
"rugged_duration_ms": 1.62,
"correlation_id": "0lCSCIy6ug9"
}
More details in #36452 (comment 245801832)
Edited by Mark Lapierre