Allow to configure the liveness and readiness probe for gitaly

What does this MR do?

Allows livenessProbe and readinessProbe to be configured for gitaly

Followed similar MRs !985 (merged) for unicorn and !1021 (merged) for gitlab-shell

Related issues

#1576

Checklist

See Definition of done.

For anything in this list which will not be completed, please provide a reason in the MR discussion.

Required

• Merge Request Title and Description are up to date, accurate, and descriptive
• MR targeting the appropriate branch
• MR has a green pipeline on GitLab.com
• When ready for review, MR is labeled "~workflow::ready for review" per the Distribution MR workflow

Expected (please provide an explanation if not completing)

• Test plan indicating conditions for success has been posted and passes
• Documentation created/updated
• Tests added
• [ ] Integration tests added to GitLab QA No functional changes made.
• [ ] Equivalent MR/issue for omnibus-gitlab opened Doesn't apply to omnibus.
• [ ] Validate potential values for new configuration settings. Formats such as integer 10, duration 10s, URI scheme://user:passwd@host:port may require quotation or other special handling when rendered in a template and written to a configuration file. Doesn't apply is applied directly to k8s manifest.

Tested with:

> helm template . --set certmanager-issuer.email=foo@bar.com --set gitlab.gitaly.statefulset.livenessProbe.initialDelaySeconds=42 --set gitlab.gitaly.statefulset.livenessProbe.periodSeconds=66 --set gitlab.gitaly.statefulset.livenessProbe.timeoutSeconds=99 --set gitlab.gitaly.statefulset.livenessProbe.successThreshold=12 --set gitlab.gitaly.statefulset.livenessProbe.failureThreshold=13 --set gitlab.gitaly.statefulset.readinessProbe.initialDelaySeconds=43 --set gitlab.gitaly.statefulset.readinessProbe.periodSeconds=67 --set gitlab.gitaly.statefulset.readinessProbe.timeoutSeconds=98 --set gitlab.gitaly.statefulset.readinessProbe.successThreshold=14 --set gitlab.gitaly.statefulset.readinessProbe.failureThreshold=7 -s charts/gitlab/charts/gitaly/templates/statefulset.yml
---
# Source: gitlab/charts/gitlab/charts/gitaly/templates/statefulset.yml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-gitaly
  namespace: default
  labels:
    app: gitaly
    chart: gitaly-6.5.1
    release: release-name
    heritage: Helm

spec:
  selector:
    matchLabels:
      app: gitaly
      release: release-name

  serviceName: release-name-gitaly
  replicas: 1
  podManagementPolicy: Parallel
  template:
    metadata:
      labels:
        app: gitaly
        chart: gitaly-6.5.1
        release: release-name
        heritage: Helm


      annotations:
        checksum/config: c2bc17673f58a28112bd3bd1b8d6a4a706bfbc6325d35e438a0a9fe27e79c8a6
        gitlab.com/prometheus_scrape: "true"
        gitlab.com/prometheus_port: "9236"
        gitlab.com/prometheus_path: /metrics
        prometheus.io/scrape: "true"
        prometheus.io/port: "9236"
        prometheus.io/path: /metrics
    spec:
      terminationGracePeriodSeconds: 30
      initContainers:

        - name: certificates
          image: "registry.gitlab.com/gitlab-org/build/cng/alpine-certificates:20191127-r2@sha256:367d437d024d7647432d67fb2442e3e5723af5930bad77d3535f4f8f4f8630d9"
          env:


          volumeMounts:
          - name: etc-ssl-certs
            mountPath: /etc/ssl/certs
            readOnly: false
          - name: etc-pki-ca-trust-extracted-pem
            mountPath: /etc/pki/ca-trust/extracted/pem
            readOnly: false
          resources:
            requests:
              cpu: 50m
        - name: configure
          command: ['sh', '/config/configure']
          image: "registry.gitlab.com/gitlab-org/cloud-native/mirror/images/busybox:latest"
          env:


          volumeMounts:

          - name: gitaly-config
            mountPath: /config
            readOnly: true
          - name: init-gitaly-secrets
            mountPath: /init-config
            readOnly: true
          - name: gitaly-secrets
            mountPath: /init-secrets
            readOnly: false
          resources:
            requests:
              cpu: 50m
      securityContext:
        runAsUser: 1000
        fsGroup: 1000
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            podAffinityTerm:
              topologyKey: "kubernetes.io/hostname"
              labelSelector:
                matchLabels:
                  app: gitaly
                  release: release-name

      automountServiceAccountToken: false
      containers:

        - name: gitaly
          image: "registry.gitlab.com/gitlab-org/build/cng/gitaly:master"
          ports:
            - containerPort: 8075
              name: grpc-gitaly
            - containerPort: 9236
              name: http-metrics
          env:
            - name: CONFIG_TEMPLATE_DIRECTORY
              value: '/etc/gitaly/templates'
            - name: CONFIG_DIRECTORY
              value: '/etc/gitaly'
            - name: GITALY_CONFIG_FILE
              value: '/etc/gitaly/config.toml'
            - name: SSL_CERT_DIR
              value: '/etc/ssl/certs'



          volumeMounts:

            - name: etc-ssl-certs
              mountPath: /etc/ssl/certs/
              readOnly: true
            - name: etc-pki-ca-trust-extracted-pem
              mountPath: /etc/pki/ca-trust/extracted/pem
              readOnly: true
            - name: gitaly-config
              mountPath: '/etc/gitaly/templates'
            - name: gitaly-secrets
              mountPath: '/etc/gitlab-secrets'
              readOnly: true
            - name: repo-data
              mountPath: '/home/git/repositories'
          livenessProbe:
            exec:
              command:
              - /scripts/healthcheck
            initialDelaySeconds: 42
            periodSeconds: 66
            timeoutSeconds: 99
            successThreshold: 12
            failureThreshold: 13
          readinessProbe:
            exec:
              command:
              - /scripts/healthcheck
            initialDelaySeconds: 43
            periodSeconds: 67
            timeoutSeconds: 98
            successThreshold: 14
            failureThreshold: 7
          resources:
            requests:
              cpu: 100m
              memory: 200Mi
      volumes:

      - name: gitaly-config
        configMap:
          name: release-name-gitaly
      - name: gitaly-secrets
        emptyDir:
          medium: "Memory"
      - name: init-gitaly-secrets
        projected:
          defaultMode: 0440
          sources:
          - secret:
              name: "release-name-gitaly-secret"
              items:
                - key: "token"
                  path: "gitaly_token"
          - secret:
              name: "release-name-gitlab-shell-secret"
              items:
                - key: "secret"
                  path: ".gitlab_shell_secret"

      - name: etc-ssl-certs
        emptyDir:
          medium: "Memory"
      - name: etc-pki-ca-trust-extracted-pem
        emptyDir:
          medium: "Memory"



  volumeClaimTemplates:
    - metadata:
        name: repo-data
        labels:
          app: gitaly
          release: release-name
        annotations:
      spec:
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "50Gi"
        selector: