Failure in qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb and qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb where runner fails to be found from the API
Summary
In !31157 (diffs), runner tags were removed so that runners are created without any tags by default.
This is ok in a pristine environment where there are not a lot of runners as the runners would be retrieved in the first page of API results (with project.runners
), but in environment with a lot of untagged runners, the QA runner could only appear in subsequent API pages, thus the runner would fail to be removed.
This is what happens in https://ops.gitlab.net/gitlab-org/quality/staging/-/jobs/1188296.
Easy fix is to re-add tags to the runner.
Stack trace
1) Verify Runner registration user registers a new specific runner
Failure/Error: runner.remove_via_api!
RuntimeError:
Project gitlab-qa-sandbox-group/qa-test-2020-05-14-08-30-32-736b57b2e53fc607/project-with-ci-cd-f8d2ede6fc2b2c0b does not have a runner with a description matching qa-runner-1589445093 . Runners available: [{:id=>37397, :description=>"gitlab-shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>37398, :description=>"gitlab-shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>40786, :description=>"shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>40788, :description=>"shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>44028, :description=>"shared-runners-manager-3.gitlab.com", :ip_address=>"35.229.49.100", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>true, :status=>"online"}, {:id=>44949, :description=>"shared-runners-manager-4.gitlab.com", :ip_address=>"35.227.21.133", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>true, :status=>"online"}, {:id=>157328, :description=>"gitlab-shared-runners-manager-3.gitlab.com", :ip_address=>nil, :active=>false, :is_shared=>true, :name=>"gitlab-ci-multi-runner", :online=>false, :status=>"paused"}, {:id=>157329, :description=>"gitlab-shared-runners-manager-4.gitlab.com", :ip_address=>nil, :active=>false, :is_shared=>true, :name=>"gitlab-ci-multi-runner", :online=>false, :status=>"paused"}, {:id=>186460, :description=>"hc-shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>186461, :description=>"hc-shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>197697, :description=>"fabio-migration-runner", :ip_address=>"35.206.112.172", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>202405, :description=>nil, :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"kubernetes-cluster", :online=>nil, :status=>"not_connected"}, {:id=>202698, :description=>nil, :ip_address=>"34.72.8.181", :active=>true, :is_shared=>true, :name=>"kubernetes-cluster", :online=>false, :status=>"offline"}, {:id=>204907, :description=>"upstream-project-ae5df7d62bbce628", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204916, :description=>"upstream-project-ed3a4d881481377b", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204917, :description=>"upstream-project-7b75239b82a91424", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204918, :description=>"upstream-project-fe6d2f262d51d7b7", :ip_address=>"35.237.226.43", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204919, :description=>"upstream-project-c915f33439805d20", :ip_address=>"35.237.245.106", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204921, :description=>"upstream-project-19bd2282db5d5160", :ip_address=>"35.243.169.14", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204923, :description=>"upstream-project-a2f46e692e257587", :ip_address=>"196.52.60.49", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}]
# ./qa/resource/runner.rb:58:in `remove_via_api!'
# ./qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb:14:in `block (3 levels) in <module:QA>'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:123:in `block in run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `loop'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec_ext/rspec_ext.rb:12:in `run_with_retry'
# ./spec/spec_helper.rb:69:in `block (2 levels) in <top (required)>'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:123:in `block in run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `loop'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec_ext/rspec_ext.rb:12:in `run_with_retry'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:37:in `block (2 levels) in setup'
# ./qa/specs/runner.rb:71:in `perform'
# ./qa/scenario/template.rb:10:in `block in perform'
# ./qa/scenario/template.rb:8:in `tap'
# ./qa/scenario/template.rb:8:in `perform'
# ./qa/scenario/template.rb:33:in `perform'
# ./qa/scenario/template.rb:10:in `block in perform'
# ./qa/scenario/template.rb:8:in `tap'
# ./qa/scenario/template.rb:8:in `perform'
# ./qa/scenario/bootable.rb:28:in `launch!'
2) Verify Code coverage statistics creates an MR with code coverage statistics
Failure/Error: runner.remove_via_api!
RuntimeError:
Project gitlab-qa-sandbox-group/qa-test-2020-05-14-08-30-32-736b57b2e53fc607/project-with-ci-cd-ea9c13b23a7d1b81 does not have a runner with a description matching qa-runner-1589445206 . Runners available: [{:id=>37397, :description=>"gitlab-shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>37398, :description=>"gitlab-shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>40786, :description=>"shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>40788, :description=>"shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>44028, :description=>"shared-runners-manager-3.gitlab.com", :ip_address=>"35.229.49.100", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>true, :status=>"online"}, {:id=>44949, :description=>"shared-runners-manager-4.gitlab.com", :ip_address=>"35.227.21.133", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>true, :status=>"online"}, {:id=>157328, :description=>"gitlab-shared-runners-manager-3.gitlab.com", :ip_address=>nil, :active=>false, :is_shared=>true, :name=>"gitlab-ci-multi-runner", :online=>false, :status=>"paused"}, {:id=>157329, :description=>"gitlab-shared-runners-manager-4.gitlab.com", :ip_address=>nil, :active=>false, :is_shared=>true, :name=>"gitlab-ci-multi-runner", :online=>false, :status=>"paused"}, {:id=>186460, :description=>"hc-shared-runners-manager-1.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>186461, :description=>"hc-shared-runners-manager-2.gitlab.com", :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>197697, :description=>"fabio-migration-runner", :ip_address=>"35.206.112.172", :active=>true, :is_shared=>true, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>202405, :description=>nil, :ip_address=>nil, :active=>true, :is_shared=>true, :name=>"kubernetes-cluster", :online=>nil, :status=>"not_connected"}, {:id=>202698, :description=>nil, :ip_address=>"34.72.8.181", :active=>true, :is_shared=>true, :name=>"kubernetes-cluster", :online=>false, :status=>"offline"}, {:id=>204907, :description=>"upstream-project-ae5df7d62bbce628", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204916, :description=>"upstream-project-ed3a4d881481377b", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204917, :description=>"upstream-project-7b75239b82a91424", :ip_address=>"35.243.174.124", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204918, :description=>"upstream-project-fe6d2f262d51d7b7", :ip_address=>"35.237.226.43", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204919, :description=>"upstream-project-c915f33439805d20", :ip_address=>"35.237.245.106", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204921, :description=>"upstream-project-19bd2282db5d5160", :ip_address=>"35.243.169.14", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}, {:id=>204923, :description=>"upstream-project-a2f46e692e257587", :ip_address=>"196.52.60.49", :active=>true, :is_shared=>false, :name=>"gitlab-runner", :online=>false, :status=>"offline"}]
# ./qa/resource/runner.rb:58:in `remove_via_api!'
# ./qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb:32:in `block (3 levels) in <module:QA>'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:123:in `block in run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `loop'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec_ext/rspec_ext.rb:12:in `run_with_retry'
# ./spec/spec_helper.rb:69:in `block (2 levels) in <top (required)>'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:123:in `block in run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `loop'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:110:in `run'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec_ext/rspec_ext.rb:12:in `run_with_retry'
# /usr/local/bundle/gems/rspec-retry-0.6.1/lib/rspec/retry.rb:37:in `block (2 levels) in setup'
# ./qa/specs/runner.rb:71:in `perform'
# ./qa/scenario/template.rb:10:in `block in perform'
# ./qa/scenario/template.rb:8:in `tap'
# ./qa/scenario/template.rb:8:in `perform'
# ./qa/scenario/template.rb:33:in `perform'
# ./qa/scenario/template.rb:10:in `block in perform'
# ./qa/scenario/template.rb:8:in `tap'
# ./qa/scenario/template.rb:8:in `perform'
# ./qa/scenario/bootable.rb:28:in `launch!'
Screenshot / HTML page
We can see that the runner do exists in the UI, we just fail to find it in the first API page of results: