Kubernetes (Helm chart), docker private registry with self signed cert - x509: certificate signed by unknown authority

Problem to solve

Hello, our company have a self run a Gitlab instance with self signed certificate and I am trying to install self hosted Gitlab runner inside Kubernetes (Helm chart) and use images from self hosted docker registry with self signed certificate. I read many posts, but I cannot find solution to this.

Further details

Gitlab runner takes cert chain from Kubernetes secret and successfully register, I can see it in CI/CD. When I run a pipeline, it take a job and start additional pods but fail with error: Failed to pull image "10.0.xxx.yyy:5000/our/yaml:latest": rpc error: code = Unknown desc = failed to pull and unpack image "10.0.36.185:5000/our/yaml:latest": failed to resolve reference "10.0.xxx.yyy:5000/our/yaml:latest": failed to do request: Head "https://10.0.xxx.yyy:5000/v2/our/yaml/manifests/latest": x509: certificate signed by unknown authority

Steps to reproduce

Docker registry

# Self signed cert
mkdir certs
openssl req \
  -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key \
  -addext "subjectAltName = IP:10.0.xxx.yyy" \
  -x509 -days 365 -out certs/domain.crt

cat certs/domain.crt >> OUR-CHAIN.pem

sudo mkdir -p /etc/docker/certs.d/10.0.xxx.yyy:5000/
sudo cp certs/domain.crt /etc/docker/certs.d/10.0.xxx.yyy:5000/ca.crt

# Registry auth
mkdir auth

docker run \
  --entrypoint htpasswd \
  httpd:2 -Bbn gitlab "secret-password" > auth/htpasswd

sudo mkdir -p /mnt/registry

# Start registry
docker run -d \
  -p 5000:5000 \
  --restart=always \
  --name registry \
  -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
  -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
  -e "REGISTRY_AUTH=htpasswd" \
  -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
  -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
  -v /mnt/registry:/var/lib/registry \
  -v "$(pwd)"/certs:/certs \
  -v "$(pwd)"/auth:/auth \
  registry:2

# Login to registry
docker login 10.0.xxx.yyy:5000

Gitlab runner

kubectl create namespace gitlab-runners

nano values.yaml
###
imagePullPolicy: IfNotPresent
imagePullSecrets:
  - name: "rpi-dev-registry"
probeTimeoutSeconds: 60
replicas: 1
revisionHistoryLimit: 2
gitlabUrl: https://git.xxxxxx.local/
runnerRegistrationToken: "registry-token"
unregisterRunners: true
terminationGracePeriodSeconds: 3600
certsSecretName: gitlab-domain-cert
concurrent: 10
checkInterval: 30
rbac:
  create: true
  serviceAccountName: gitlab-runner-admin
  rules: []
  clusterWideAccess: false
  podSecurityPolicy:
    enabled: false
    resourceNames:
    - gitlab-runner
  imagePullSecrets: [rpi-dev-registry]

metrics:
  enabled: true
  portName: metrics
  port: 9252
serviceMonitor:
    enabled: false
service:
  enabled: false
  type: ClusterIP

runners:
config: |
    [[runners]]
      tls_verify = false
      environment = ["DOCKER_AUTH_CONFIG={\"auths\": {\"10.0.xxx.yyy:5000\": {\"auth\": \"base64-encoded-password==\"}}}"]
      [runners.kubernetes]
        helper_image = "gitlab/gitlab-runner-helper:arm64-latest"
        namespace = "gitlab-runners"
        image = "ubuntu-job:latest"
        image_pull_secrets = ["rpi-dev-registry"]
      [[runners.kubernetes.volumes.host_path]]
        name = "cert"
        mount_path = "/etc/ssl/certs/our.crt"
        read_only = true
        host_path = "/home/askala/OUR-CHAIN.pem"
    tags: "python,yaml"
    name: "DEV_RPI"
    serviceAccountName: gitlab-runner-admin
securityContext:
  runAsUser: 100
  fsGroup: 65533
###

nano service-account.yaml
###
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-runner-admin
  namespace: gitlab-runners
apiVersion: rbac.authorization.k8s.io/v1
kind: "ClusterRole"
metadata:
  name: gitlab-runner-admin
  namespace: gitlab-runners
rules:
- apiGroups: ["*"]
  resources: ["*"]
  verbs: ["*"]
apiVersion: rbac.authorization.k8s.io/v1
kind: "ClusterRoleBinding"
metadata:
  name: gitlab-runner-admin
  namespace: gitlab-runners
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: "ClusterRole"
  name: gitlab-runner-admin
subjects:
- kind: ServiceAccount
  name: gitlab-runner-admin
  namespace: gitlab-runners
###
kubectl apply -f service-account.yaml


kubectl create secret generic gitlab-domain-cert \
  --namespace gitlab-runners \
  --from-file=git.xxxx.local.crt=OUR-CHAIN.pem

# deploy
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
helm install --namespace gitlab-runners gitlab-runner -f values.yaml gitlab/gitlab-runner