Multiple QA pipelines failing due to potential Docker Login issue
Summary
Multiple pipelines (Staging and Preprod so far) have started failing today due to a docker pull issue. This is possible due to a failed docker login and may be related to a recently merged MR - gitlab-qa!468 (merged)
Stack trace
$ gitlab-qa $SCENARIO $RELEASE -- $TEST_OPTS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
Running test suite `Test::Instance::All` for gitlab-ee
Docker shell command: `docker run -t --rm --net=bridge --env QA_REMOTE_GRID_USERNAME="$QA_REMOTE_GRID_USERNAME" --env QA_REMOTE_GRID_ACCESS_KEY="$QA_REMOTE_GRID_ACCESS_KEY" --env QA_REMOTE_GRID_PROTOCOL="$QA_REMOTE_GRID_PROTOCOL" --env GITLAB_ADMIN_USERNAME="$GITLAB_ADMIN_USERNAME" --env GITLAB_ADMIN_PASSWORD="$GITLAB_ADMIN_PASSWORD" --env GITLAB_USERNAME="$GITLAB_USERNAME" --env GITLAB_PASSWORD="$GITLAB_PASSWORD" --env GITLAB_FORKER_USERNAME="$GITLAB_FORKER_USERNAME" --env GITLAB_FORKER_PASSWORD="$GITLAB_FORKER_PASSWORD" --env GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_ACCESS_TOKEN" --env GITLAB_QA_ADMIN_ACCESS_TOKEN="$GITLAB_QA_ADMIN_ACCESS_TOKEN" --env GITHUB_ACCESS_TOKEN="$GITHUB_ACCESS_TOKEN" --env GCLOUD_ACCOUNT_EMAIL="$GCLOUD_ACCOUNT_EMAIL" --env GCLOUD_ACCOUNT_KEY="$GCLOUD_ACCOUNT_KEY" --env CLOUDSDK_CORE_PROJECT="$CLOUDSDK_CORE_PROJECT" --env GCLOUD_REGION="$GCLOUD_REGION" --env SIGNUP_DISABLED="$SIGNUP_DISABLED" --env QA_PRAEFECT_REPOSITORY_STORAGE="$QA_PRAEFECT_REPOSITORY_STORAGE" --env QA_DEBUG="$QA_DEBUG" --env QA_LOG_PATH="$QA_LOG_PATH" --env QA_CAN_TEST_ADMIN_FEATURES="$QA_CAN_TEST_ADMIN_FEATURES" --env QA_CAN_TEST_GIT_PROTOCOL_V2="$QA_CAN_TEST_GIT_PROTOCOL_V2" --env QA_CAN_TEST_PRAEFECT="$QA_CAN_TEST_PRAEFECT" --env GITLAB_QA_PASSWORD_1="$GITLAB_QA_PASSWORD_1" --env GITLAB_QA_PASSWORD_2="$GITLAB_QA_PASSWORD_2" --env GITHUB_USERNAME="$GITHUB_USERNAME" --env GITHUB_PASSWORD="$GITHUB_PASSWORD" --env CI="$CI" --env CI_RUNNER_ID="$CI_RUNNER_ID" --env CI_SERVER_HOST="$CI_SERVER_HOST" --env CI_NODE_TOTAL="$CI_NODE_TOTAL" --env GITLAB_CI="$GITLAB_CI" --env CI_SLACK_WEBHOOK_URL="$CI_SLACK_WEBHOOK_URL" --env GITLAB_QA_FORMLESS_LOGIN_TOKEN="$GITLAB_QA_FORMLESS_LOGIN_TOKEN" --env JIRA_ADMIN_USERNAME="$JIRA_ADMIN_USERNAME" --env JIRA_ADMIN_PASSWORD="$JIRA_ADMIN_PASSWORD" --volume /var/run/docker.sock:/var/run/docker.sock:z --volume /builds/gitlab-org/quality/staging/gitlab-qa-run-2020-06-02-15-22-04-741ae77d/gitlab-ee-qa-e8776b4a:/home/gitlab/qa/tmp:z --name gitlab-ee-qa-e8776b4a dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee-qa:13.1-8324d2ed53e Test::Instance::All https://staging.gitlab.com -- --tag reliable --format RspecJunitFormatter --out "tmp/rspec-1255905.xml" --format html --out "tmp/rspec-$(echo $CI_JOB_NAME | sed 's/[ /]/_/g').htm" --color --format documentation`
/usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/shellout.rb:31:in `block in execute!': Docker command `docker run -t --rm --net=bridge --env QA_REMOTE_GRID_USERNAME="$QA_REMOTE_GRID_USERNAME" --env QA_REMOTE_GRID_ACCESS_KEY="$QA_REMOTE_GRID_ACCESS_KEY" --env QA_REMOTE_GRID_PROTOCOL="$QA_REMOTE_GRID_PROTOCOL" --env GITLAB_ADMIN_USERNAME="$GITLAB_ADMIN_USERNAME" --env GITLAB_ADMIN_PASSWORD="$GITLAB_ADMIN_PASSWORD" --env GITLAB_USERNAME="$GITLAB_USERNAME" --env GITLAB_PASSWORD="$GITLAB_PASSWORD" --env GITLAB_FORKER_USERNAME="$GITLAB_FORKER_USERNAME" --env GITLAB_FORKER_PASSWORD="$GITLAB_FORKER_PASSWORD" --env GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_ACCESS_TOKEN" --env GITLAB_QA_ADMIN_ACCESS_TOKEN="$GITLAB_QA_ADMIN_ACCESS_TOKEN" --env GITHUB_ACCESS_TOKEN="$GITHUB_ACCESS_TOKEN" --env GCLOUD_ACCOUNT_EMAIL="$GCLOUD_ACCOUNT_EMAIL" --env GCLOUD_ACCOUNT_KEY="$GCLOUD_ACCOUNT_KEY" --env CLOUDSDK_CORE_PROJECT="$CLOUDSDK_CORE_PROJECT" --env GCLOUD_REGION="$GCLOUD_REGION" --env SIGNUP_DISABLED="$SIGNUP_DISABLED" --env QA_PRAEFECT_REPOSITORY_STORAGE="$QA_PRAEFECT_REPOSITORY_STORAGE" --env QA_DEBUG="$QA_DEBUG" --env QA_LOG_PATH="$QA_LOG_PATH" --env QA_CAN_TEST_ADMIN_FEATURES="$QA_CAN_TEST_ADMIN_FEATURES" --env QA_CAN_TEST_GIT_PROTOCOL_V2="$QA_CAN_TEST_GIT_PROTOCOL_V2" --env QA_CAN_TEST_PRAEFECT="$QA_CAN_TEST_PRAEFECT" --env GITLAB_QA_PASSWORD_1="$GITLAB_QA_PASSWORD_1" --env GITLAB_QA_PASSWORD_2="$GITLAB_QA_PASSWORD_2" --env GITHUB_USERNAME="$GITHUB_USERNAME" --env GITHUB_PASSWORD="$GITHUB_PASSWORD" --env CI="$CI" --env CI_RUNNER_ID="$CI_RUNNER_ID" --env CI_SERVER_HOST="$CI_SERVER_HOST" --env CI_NODE_TOTAL="$CI_NODE_TOTAL" --env GITLAB_CI="$GITLAB_CI" --env CI_SLACK_WEBHOOK_URL="$CI_SLACK_WEBHOOK_URL" --env GITLAB_QA_FORMLESS_LOGIN_TOKEN="$GITLAB_QA_FORMLESS_LOGIN_TOKEN" --env JIRA_ADMIN_USERNAME="$JIRA_ADMIN_USERNAME" --env JIRA_ADMIN_PASSWORD="$JIRA_ADMIN_PASSWORD" --volume /var/run/docker.sock:/var/run/docker.sock:z --volume /builds/gitlab-org/quality/staging/gitlab-qa-run-2020-06-02-15-22-04-741ae77d/gitlab-ee-qa-e8776b4a:/home/gitlab/qa/tmp:z --name gitlab-ee-qa-e8776b4a dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee-qa:13.1-8324d2ed53e Test::Instance::All https://staging.gitlab.com -- --tag reliable --format RspecJunitFormatter --out "tmp/rspec-1255905.xml" --format html --out "tmp/rspec-$(echo $CI_JOB_NAME | sed 's/[ /]/_/g').htm" --color --format documentation` failed! (Gitlab::QA::Docker::Shellout::StatusError)
from /usr/local/lib/ruby/2.6.0/open3.rb:219:in `popen_run'
from /usr/local/lib/ruby/2.6.0/open3.rb:208:in `popen2e'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/shellout.rb:19:in `execute!'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/command.rb:36:in `execute!'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/engine.rb:26:in `block in run'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/engine.rb:20:in `tap'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/docker/engine.rb:20:in `run'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/component/specs.rb:26:in `perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:8:in `block in perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:6:in `tap'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:6:in `perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/test/instance/deployment_base.rb:26:in `perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:8:in `block in perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:6:in `tap'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/scenario/template.rb:6:in `perform'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/lib/gitlab/qa/runner.rb:42:in `run'
from /usr/local/bundle/gems/gitlab-qa-5.13.4/exe/gitlab-qa:8:in `<top (required)>'
from /usr/local/bundle/bin/gitlab-qa:23:in `load'
from /usr/local/bundle/bin/gitlab-qa:23:in `<main>'
Unable to find image 'dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee-qa:13.1-8324d2ed53e' locally
docker: Error response from daemon: Get https://dev.gitlab.org:5005/v2/gitlab/omnibus-gitlab/gitlab-ee-qa/manifests/13.1-8324d2ed53e: denied: access forbidden.