Skip to content

Add support for podLabels for statefulSet and deployment

Terri Churequested to merge
tchu-add-pod-labels-for-deployment-and-statefulset into main

What does this MR do and why?

References:

This change adds support for configuring custom labels and annotations on pods at both the global level and for specific deployment types (deployment and statefulSet).

Previously, users could only set global pod annotations. Now they can also set global pod labels, plus deployment-specific and statefulSet-specific labels and annotations. This gives users more flexibility to organize and identify their pods with custom metadata.

The implementation includes:

  • New configuration options in the values file for podLabels globally, and podLabels/podAnnotations for both deployment and statefulSet sections
  • Updates to the Helm templates to apply these new labels and annotations to the generated pods
  • A new helper method to extract annotations from templates for testing
  • Comprehensive tests to verify the new functionality works correctly

This enhancement allows better pod organization and integration with monitoring, security, and other systems that rely on Kubernetes labels and annotations.

How to set up and validate locally

helm template . --set deployment.podLabels."type"="zoekt" --set statefulSet.podLabels."type"="zoekt"   --set indexer.internalApi.gitlabUrl=https://gitlab.example.com --set indexer.internalApi.secretName=secretName --set indexer.internalApi.secretKey=secretKey
Click to expand output 
---
# Source: gitlab-zoekt/templates/deployment-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-gitlab-zoekt-deployment-pdb
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
spec:
  maxUnavailable: "10%"
  selector:
    matchLabels:
      app.kubernetes.io/name: gitlab-zoekt-gateway
      app.kubernetes.io/instance: release-name
---
# Source: gitlab-zoekt/templates/statefulset-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-gitlab-zoekt-statefulset-pdb
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
spec:
  maxUnavailable: "10%"
  selector:
    matchLabels:
      app.kubernetes.io/name: gitlab-zoekt
      app.kubernetes.io/instance: release-name
---
# Source: gitlab-zoekt/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-gitlab-zoekt
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
---
# Source: gitlab-zoekt/templates/deployment-nginx-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-gitlab-zoekt-gateway-nginx-conf
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
data:
  nginx.conf: |
    events {
        use epoll;
    }

    http {
        http2 on;
        server {
            listen       8080;
            server_tokens off;

            proxy_http_version 1.1;
            proxy_set_header Host            $host;
            proxy_set_header X-Forwarded-For $remote_addr;
            proxy_set_header Authorization $http_authorization;
            proxy_set_header Gitlab-Zoekt-Api-Request $http_gitlab_zoekt_api_request;

            proxy_next_upstream off;

            resolver kube-dns.kube-system.svc.cluster.local;

            set $backend_gateway release-name-gitlab-zoekt.default.svc.cluster.local:8080;

            location = /health {
              return 200;
            }

            location ~ ^/(api|indexer|webserver)/.* {
              set $target_host http://$backend_gateway;

              proxy_pass $target_host;
              proxy_read_timeout 30m;
            }

            # HTTP webserver and indexer nodes endpoints
            location ~ ^/nodes/([^/]+)/(api|indexer|webserver)/(.*) {
              set $target_host http://$1:8080/$2/$3;

              proxy_pass $target_host;
              proxy_read_timeout 30m;
            }

            # gRPC catch all
            location / {
              # If X-Forwarded-Path header is provided, extract the fully qualified domain name
              # Format should be: /nodes/full.qualified.domain.name
              if ($http_x_forwarded_path ~* "^/nodes/(.+)$") {
                set $node_name $1:8080;
              }

              # Set appropriate grpc target scheme based on TLS configuration
              set $grpc_target grpc://$node_name;

              # Forward to the target node
              grpc_pass $grpc_target;

              grpc_set_header Content-Type application/grpc;

              grpc_read_timeout 1m;
              grpc_send_timeout 1m;
            }
        }
    }
---
# Source: gitlab-zoekt/templates/nginx-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-gitlab-zoekt-nginx-conf
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
data:
  nginx.conf: |
    events {
        use epoll;
    }

    http {
        upstream indexer {
          server 127.0.0.1:6065;
        }

        upstream webserver {
          server 127.0.0.1:6070;
        }
        http2 on;

        server {
            listen       8080;
            server_tokens off;
            proxy_next_upstream off;
            proxy_set_header Authorization $http_authorization;
            proxy_set_header Gitlab-Zoekt-Api-Request $http_gitlab_zoekt_api_request;

            # Internal gateway healthcheck
            location = /health {
              auth_basic off;
              return 200;
            }

            # Webserver healthcheck
            location = / {
              auth_basic off;
              proxy_pass http://webserver/;
            }

            # Webserver metrics
            location = /metrics {
              auth_basic off;
              proxy_pass http://webserver/metrics;
            }

            # Webserver metrics (alias)
            location = /webserver/metrics {
              auth_basic off;
              proxy_pass http://webserver/metrics;
            }

            # Indexer metrics
            location = /indexer/metrics {
              auth_basic off;
              proxy_pass http://indexer;
            }

            location /indexer {
              proxy_pass http://indexer;
              proxy_read_timeout 30m;
            }

            location /webserver {
                proxy_pass http://webserver;
                proxy_read_timeout 1m;
            }

            # HTTP webserver endpoint
            location /api {
                proxy_pass http://webserver;
                proxy_read_timeout 1m;
            }

            # gRPC catch all
            location / {
                grpc_pass grpc://webserver;
                grpc_set_header Content-Type application/grpc;
                grpc_read_timeout 1m;
                grpc_send_timeout 1m;
            }
        }
    }
---
# Source: gitlab-zoekt/templates/svc-backend.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-gitlab-zoekt
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
    
spec:
  clusterIP: None
  ports:
  - port: 8080
    name: gateway
  selector:
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
---
# Source: gitlab-zoekt/templates/svc-gateway.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-gitlab-zoekt-gateway
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
    
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - port: 8080
    name: gateway
  selector:
    app.kubernetes.io/name: gitlab-zoekt-gateway
    app.kubernetes.io/instance: release-name
---
# Source: gitlab-zoekt/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-gitlab-zoekt-gateway
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: gitlab-zoekt-gateway
      app.kubernetes.io/instance: release-name

  strategy:
    rollingUpdate:
      maxUnavailable: 10%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        checksum/config: 55575a1c16f763e0d280450ddce3e76543cb2265732a31b75709f35e0a122725
      labels:
        app.kubernetes.io/name: gitlab-zoekt-gateway
        app.kubernetes.io/instance: release-name
        type: zoekt
    spec:
      containers:
        - name: zoekt-external-gateway
          securityContext:
            {}
          image: nginx:1.25.5
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
              name: gateway
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
          volumeMounts:
            - name: nginx-conf
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
              readOnly: true
            - name: external-nginx-var-run
              mountPath: /var/run
            - name: external-nginx-var-cache-nginx
              mountPath: /var/cache/nginx
      volumes:
        - name: nginx-conf
          configMap:
            name: release-name-gitlab-zoekt-gateway-nginx-conf
            items:
              - key: nginx.conf
                path: nginx.conf
        - name: external-nginx-var-run
          emptyDir: {}
        - name: external-nginx-var-cache-nginx
          emptyDir: {}
---
# Source: gitlab-zoekt/templates/stateful_sets.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-gitlab-zoekt
  namespace: default
  labels:
    helm.sh/chart: gitlab-zoekt-3.4.0
    app.kubernetes.io/name: gitlab-zoekt
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.4.4"
    app.kubernetes.io/managed-by: Helm
    
    
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: gitlab-zoekt
      app.kubernetes.io/instance: release-name
  serviceName: release-name-gitlab-zoekt
  replicas: 1
  updateStrategy:
    rollingUpdate:
      partition: 0
    type: RollingUpdate
  template:
    metadata:
      annotations:
        checksum/config: a9f289b7193fab70d65440c9c49d740b3aa2862649d93b732a95c8e513c52d11        
      labels:
        app.kubernetes.io/name: gitlab-zoekt
        app.kubernetes.io/instance: release-name
        type: zoekt
    spec:
      serviceAccountName: release-name-gitlab-zoekt
      securityContext:
        fsGroup: 1000
      terminationGracePeriodSeconds: 60
      containers:
        - name: zoekt-indexer
          securityContext:
            runAsGroup: 1000
            runAsUser: 1000
          image: registry.gitlab.com/gitlab-org/build/cng/gitlab-zoekt:v1.4.4
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 6065
            name: indexer
          livenessProbe:
            httpGet:
              path: /indexer/health
              port: 6065
          readinessProbe:
            httpGet:
              path: /indexer/health
              port: 6065
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
          volumeMounts:
          - name: zoekt-index
            mountPath: /data/index
          - name: internal-api-secret
            readOnly: true
            mountPath: /.gitlab_shell_secret
            subPath: .gitlab_shell_secret
          env:
          - name: GITLAB_ZOEKT_MODE
            value: "indexer"
          - name: GITLAB_URL
            value: https://gitlab.example.com
          - name: SERVICE_URL
            value: http://release-name-gitlab-zoekt-gateway.default.svc.cluster.local:8080
        - name: zoekt-webserver
          securityContext:
            runAsGroup: 1000
            runAsUser: 1000
          image: registry.gitlab.com/gitlab-org/build/cng/gitlab-zoekt:v1.4.4
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 6070
              name: webserver
          livenessProbe:
            httpGet:
              path: /healthz
              port: 6070
          readinessProbe:
            httpGet:
              path: /healthz
              port: 6070
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
          volumeMounts:
          - name: zoekt-index
            mountPath: /data/index
          - name: internal-api-secret
            readOnly: true
            mountPath: /.gitlab_shell_secret
            subPath: .gitlab_shell_secret
          env:
          - name: GITLAB_ZOEKT_MODE
            value: "webserver"
        - name: zoekt-internal-gateway
          securityContext:
            {}
          image: nginx:1.25.5
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
              name: gateway
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
          volumeMounts:
            - name: nginx-conf
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
              readOnly: true
            - name: internal-nginx-var-run
              mountPath: /var/run
            - name: internal-nginx-var-cache-nginx
              mountPath: /var/cache/nginx
      volumes:
        - name: nginx-conf
          configMap:
            name: release-name-gitlab-zoekt-nginx-conf
            items:
              - key: nginx.conf
                path: nginx.conf
        - name: internal-nginx-var-run
          emptyDir: {}
        - name: internal-nginx-var-cache-nginx
          emptyDir: {}
        - name: internal-api-secret
          secret:
            secretName: secretName
            items:
              - key: secretKey
                path: ".gitlab_shell_secret"
  volumeClaimTemplates:
  - metadata:
      name: zoekt-index
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName:  
      resources:
        requests:
          storage: 2Gi

testing in gitlab chart

I have a directory called zoekt-deploys that contains:

  • zoekt-deploys/gitlab-chart
  • zoekt-deploys/gitlab-zoekt

in gitlab-chart, change the version/repository in Chart.yaml for gitlab-zoekt

- name: gitlab-zoekt
  version: 3.5.0
  repository: file://../gitlab-zoekt
  condition: gitlab-zoekt.install
`gitlab-chart-test-values.yml` 
# Minimal GitLab configuration for testing
global:
  hosts:
    domain: gitlab.example.com
  ingress:
    configureCertmanager: false
    tls:
      enabled: false
  certmanager:
    installCertManager: false
  redis:
    host: dummy-redis-host.example.com
    password:
      enabled: false

# Add this section
certmanager-issuer:
  email: tchu@gitlab.com

# Enable gitlab-zoekt for testing
gitlab-zoekt:
  install: true

  # Required configuration
  indexer:
    internalApi:
      gitlabUrl: https://gitlab.example.com
      secretName: gitlab-zoekt-secret
      secretKey: shared_secret

  # Test global pod labels and annotations
  podLabels:
    team: "search"
    environment: "production"
    global-label: "global-value"

  podAnnotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8080"
    global-annotation: "global-value"

  # Test deployment-specific labels and annotations
  deployment:
    podLabels:
      component: "gateway"
      tier: "frontend"
      environment: "staging"  # Should override global
      deployment-label: "deployment-value"

    podAnnotations:
      nginx.ingress.kubernetes.io/rewrite-target: "/"
      prometheus.io/port: "9090"  # Should override global
      deployment-annotation: "deployment-value"

  # Test statefulSet-specific labels and annotations
  statefulSet:
    podLabels:
      component: "indexer-webserver"
      tier: "backend"
      environment: "development"  # Should override global
      statefulset-label: "statefulset-value"

    podAnnotations:
      backup.enabled: "true"
      prometheus.io/port: "9091"  # Should override global
      statefulset-annotation: "statefulset-value"

# Disable other components to speed up testing
gitlab:
  install: false
postgresql:
  install: false
redis:
  install: false
nginx-ingress:
  enabled: false
prometheus:
  install: false
gitlab-runner:
  install: false
helm template test-gitlab . -f tmp/gitlab-chart-test-values.yaml > tmp/test-output.yaml

create this script at tmp/verify_labels.sh (mine is at /Users/terrichu/projects/zoekt-deploys/gitlab-chart/tmp/verify_labels.sh)

#!/bin/bash
set -e

echo "🔍 Testing gitlab-zoekt pod labels in main chart..."

# Generate template
echo "📋 Generating template..."
helm template test-gitlab . -f tmp/gitlab-chart-test-values.yaml > tmp/test-output.yaml

echo -e "\n✅ Deployment Pod Labels:"
yq 'select(.kind == "Deployment" and .metadata.name | test("gitlab-zoekt.*gateway")) | .spec.template.metadata.labels | with_entries(select(.key | test("^(team|environment|component|tier|.*-label)$")))' tmp/test-output.yaml

echo -e "\n✅ StatefulSet Pod Labels:"
yq 'select(.kind == "StatefulSet" and .metadata.name | test("gitlab-zoekt")) | .spec.template.metadata.labels | with_entries(select(.key | test("^(team|environment|component|tier|.*-label)$")))' tmp/test-output.yaml

echo -e "\n✅ Deployment Pod Annotations:"
yq 'select(.kind == "Deployment" and .metadata.name | test("gitlab-zoekt.*gateway")) | .spec.template.metadata.annotations | with_entries(select(.key | test("prometheus|nginx|.*-annotation|backup")))' tmp/test-output.yaml

echo -e "\n✅ StatefulSet Pod Annotations:"
yq 'select(.kind == "StatefulSet" and .metadata.name | test("gitlab-zoekt")) | .spec.template.metadata.annotations | with_entries(select(.key | test("prometheus|nginx|.*-annotation|backup")))' tmp/test-output.yaml

echo -e "\n🎉 Verification complete! Check the output above for expected labels and annotations."

run the script

➜ ./tmp/verify-labels.sh
🔍 Testing gitlab-zoekt pod labels in main chart...
📋 Generating template...
load.go:138: Warning: Dependency locking is handled in Chart.lock since apiVersion "v2". We recommend migrating to Chart.lock.
coalesce.go:301: warning: destination for gitlab.redis.global.redis.password is a table. Ignoring non-table value ()

✅ Deployment Pod Labels:
component: gateway
deployment-label: deployment-value
environment: staging
global-label: global-value
team: search
tier: frontend

✅ StatefulSet Pod Labels:
component: indexer-webserver
environment: development
global-label: global-value
statefulset-label: statefulset-value
team: search
tier: backend

✅ Deployment Pod Annotations:
deployment-annotation: deployment-value
global-annotation: global-value
nginx.ingress.kubernetes.io/rewrite-target: /
prometheus.io/port: "9090"
prometheus.io/scrape: "true"

✅ StatefulSet Pod Annotations:
backup.enabled: "true"
global-annotation: global-value
prometheus.io/port: "9091"
prometheus.io/scrape: "true"
statefulset-annotation: statefulset-value

🎉 Verification complete! Check the output above for expected labels and annotations.
Edited by Terri Chu

Merge request reports