E2E CNG failing with Helm deployment failed
E2E CNG pipelines are failing on master: https://gitlab.com/gitlab-org/gitlab/-/pipelines/1671590740
[✖] running helm deployment ... failed
Upgrading helm release 'gitlab' in namespace 'gitlab'
Deployment failed, retrying...
Error: Command 'helm upgrade --install gitlab /builds/gitlab-org/gitlab/qa/tmp/helm-cng/gitlab-3f89420cf319778195403711af12c57b4aab6511.tgz --namespace gitlab --timeout 5m --values - --wait --atomic --set gitlab.gitaly.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly --set gitlab.gitaly.image.tag=60ddd77897db28a385a4a55d16cd0047935bf0bb --set gitlab.gitlab-shell.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell --set gitlab.gitlab-shell.image.tag=v14.40.0 --set gitlab.migrations.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-toolbox-ee --set gitlab.migrations.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.toolbox.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-toolbox-ee --set gitlab.toolbox.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.sidekiq.annotations.commit=c84cb724 --set gitlab.sidekiq.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee --set gitlab.sidekiq.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.webservice.annotations.commit=c84cb724 --set gitlab.webservice.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-webservice-ee --set gitlab.webservice.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.webservice.workhorse.image=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee --set gitlab.webservice.workhorse.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --repository-cache /builds/gitlab-org/gitlab/qa/tmp/helm-cng' failed!
Release "gitlab" does not exist. Installing it now.
Error: release gitlab failed, and has been uninstalled due to atomic being set: context deadline exceeded
Upgrading helm release 'gitlab' in namespace 'gitlab'
Helm deployment failed!
For more information on troubleshooting failures, see: 'https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa/gems/gitlab-cng?ref_type=heads#troubleshooting'
Following events of Warning type present in cluster:
[
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-minio-7998bcddf-xlwqc",
"message": "Liveness probe failed: dial tcp 10.244.0.21:9000: connect: connection refused"
},More warnings
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-gitaly-0",
"message": "Startup probe failed: "
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-kas-6574f5cb47-xwljb",
"message": "Liveness probe failed: Get \"http://10.244.0.8:8151/liveness\": dial tcp 10.244.0.8:8151: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-kas-6574f5cb47-xwljb",
"message": "Readiness probe failed: Get \"http://10.244.0.8:8151/readiness\": dial tcp 10.244.0.8:8151: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-webservice-default-7cfd67d467-phbph",
"message": "Readiness probe failed: Get \"http://10.244.0.19:8080/-/readiness\": dial tcp 10.244.0.19:8080: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-sidekiq-all-in-1-v2-5d4dccd555-ndbps",
"message": "Readiness probe failed: Get \"http://10.244.0.16:3808/readiness\": dial tcp 10.244.0.16:3808: connect: connection refused"
},
{
"type": "Warning",
"reason": "BackOff",
"name": "Pod/gitlab-kas-6574f5cb47-xwljb",
"message": "Back-off restarting failed container kas in pod gitlab-kas-6574f5cb47-xwljb_gitlab(55249d7c-b538-44d3-a667-d32775503435)"
},
{
"type": "Warning",
"reason": "FailedMount",
"name": "Pod/gitlab-gitlab-shell-5886549f79-b8gln",
"message": "MountVolume.SetUp failed for volume \"sshd-config\" : configmap \"gitlab-gitlab-shell-sshd\" not found"
},
{
"type": "Warning",
"reason": "FailedMount",
"name": "Pod/gitlab-gitlab-shell-5886549f79-b8gln",
"message": "MountVolume.SetUp failed for volume \"shell-config\" : configmap \"gitlab-gitlab-shell\" not found"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-postgresql-0",
"message": "Readiness probe failed: 127.0.0.1:5432 - rejecting connections\n"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-postgresql-0",
"message": "Readiness probe failed: Get \"http://10.244.0.22:9187/\": dial tcp 10.244.0.22:9187: connect: connection refused"
},
{
"type": "Warning",
"reason": "FailedMount",
"name": "Pod/gitlab-postgresql-0",
"message": "MountVolume.SetUp failed for volume \"custom-init-scripts\" : configmap \"gitlab-postgresql-init-db\" not found"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-nginx-ingress-controller-f7557f64f-xrksv",
"message": "Readiness probe failed: HTTP probe failed with statuscode: 500"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-registry-576bf57dd8-fz9fp",
"message": "Liveness probe failed: Get \"http://10.244.0.39:5001/debug/health\": dial tcp 10.244.0.39:5001: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-registry-576bf57dd8-fz9fp",
"message": "Readiness probe failed: Get \"http://10.244.0.39:5001/debug/health\": dial tcp 10.244.0.39:5001: connect: connection refused"
},
{
"type": "Warning",
"reason": "BackOff",
"name": "Pod/gitlab-webservice-default-7cfd67d467-s6jbf",
"message": "Back-off restarting failed container dependencies in pod gitlab-webservice-default-7cfd67d467-s6jbf_gitlab(7048383f-1a1c-4408-bce4-4af50f55ef72)"
},
{
"type": "Warning",
"reason": "BackOff",
"name": "Pod/gitlab-sidekiq-all-in-1-v2-5d4dccd555-n752l",
"message": "Back-off restarting failed container dependencies in pod gitlab-sidekiq-all-in-1-v2-5d4dccd555-n752l_gitlab(a3d902ad-1eaa-4c43-b8a3-fa56ad22230a)"
},
{
"type": "Warning",
"reason": "BackOff",
"name": "Pod/gitlab-migrations-5b5334a-hgrqx",
"message": "Back-off restarting failed container migrations in pod gitlab-migrations-5b5334a-hgrqx_gitlab(63d86d28-8209-445c-ab87-9d4f3d637aaf)"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-webservice-default-7cfd67d467-s6jbf",
"message": "Readiness probe failed: Get \"http://10.244.0.37:8080/-/readiness\": dial tcp 10.244.0.37:8080: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-kas-6574f5cb47-768rk",
"message": "Liveness probe failed: Get \"http://10.244.0.32:8151/liveness\": dial tcp 10.244.0.32:8151: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-sidekiq-all-in-1-v2-5d4dccd555-n752l",
"message": "Liveness probe failed: Get \"http://10.244.0.36:3808/liveness\": dial tcp 10.244.0.36:3808: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-sidekiq-all-in-1-v2-5d4dccd555-n752l",
"message": "Readiness probe failed: Get \"http://10.244.0.36:3808/readiness\": dial tcp 10.244.0.36:3808: connect: connection refused"
},
{
"type": "Warning",
"reason": "Unhealthy",
"name": "Pod/gitlab-kas-6574f5cb47-768rk",
"message": "Readiness probe failed: Get \"http://10.244.0.32:8151/readiness\": dial tcp 10.244.0.32:8151: connect: connection refused"
},
{
"type": "Warning",
"reason": "BackOff",
"name": "Pod/gitlab-kas-6574f5cb47-768rk",
"message": "Back-off restarting failed container kas in pod gitlab-kas-6574f5cb47-768rk_gitlab(ee09a642-3e14-4702-81d9-0d8b45b05708)"
}]
=== block 'running helm deployment' error ===
Command 'helm upgrade --install gitlab /builds/gitlab-org/gitlab/qa/tmp/helm-cng/gitlab-3f89420cf319778195403711af12c57b4aab6511.tgz --namespace gitlab --timeout 5m --values - --wait --set gitlab.gitaly.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly --set gitlab.gitaly.image.tag=60ddd77897db28a385a4a55d16cd0047935bf0bb --set gitlab.gitlab-shell.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell --set gitlab.gitlab-shell.image.tag=v14.40.0 --set gitlab.migrations.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-toolbox-ee --set gitlab.migrations.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.toolbox.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-toolbox-ee --set gitlab.toolbox.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.sidekiq.annotations.commit=c84cb724 --set gitlab.sidekiq.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee --set gitlab.sidekiq.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.webservice.annotations.commit=c84cb724 --set gitlab.webservice.image.repository=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-webservice-ee --set gitlab.webservice.image.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --set gitlab.webservice.workhorse.image=registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee --set gitlab.webservice.workhorse.tag=c84cb724bccab25b356b2f9c66c752a4842c7675 --repository-cache /builds/gitlab-org/gitlab/qa/tmp/helm-cng' failed!
Release "gitlab" does not exist. Installing it now.
Error: context deadline exceeded
=== block 'running helm deployment' error ===