feat: add endpoint labels to pending and status notification metrics (c501105d) · Commits · GitLab.org / container-registry

notifications/metrics.go

+8 −7

Original line number	Diff line number	Diff line
		@@ -13,7 +13,7 @@ import (

		var (
		eventsCounter *prometheus.CounterVec
		pendingGauge prometheus.Gauge
		pendingGauge *prometheus.GaugeVec
		statusCounter *prometheus.CounterVec
		errorCounter *prometheus.CounterVec
		)
		@@ -55,13 +55,14 @@ func registerMetrics(registerer prometheus.Registerer) {
		[]string{eventsTypeLabel, eventsActionLabel, eventsArtifactLabel, eventsEndpointLabel},
		)

		pendingGauge = prometheus.NewGauge(
		pendingGauge = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
		Namespace: metrics.NamespacePrefix,
		Subsystem: subsystem,
		Name: pendingGaugeName,
		Help: pendingGaugeDesc,
		},
		[]string{eventsEndpointLabel},
		)

		statusCounter = prometheus.NewCounterVec(
		@@ -71,7 +72,7 @@ func registerMetrics(registerer prometheus.Registerer) {
		Name: statusCounterName,
		Help: statusCounterDesc,
		},
		[]string{statusCodeLabel},
		[]string{statusCodeLabel, eventsEndpointLabel},
		)

		errorCounter = prometheus.NewCounterVec(
		@@ -157,7 +158,7 @@ func (emsl endpointMetricsHTTPStatusListener) success(status int, event Event)
		actual.(*atomic.Int64).Add(1)
		emsl.successes.Add(1)

		statusCounter.WithLabelValues(key).Inc()
		statusCounter.WithLabelValues(key, emsl.endpoint).Inc()
		eventsCounter.WithLabelValues("Successes", event.Action, event.artifact(), emsl.endpoint).Inc()
		}

		@@ -167,7 +168,7 @@ func (emsl endpointMetricsHTTPStatusListener) failure(status int, event Event)
		actual.(*atomic.Int64).Add(1)
		emsl.failures.Add(1)

		statusCounter.WithLabelValues(key).Inc()
		statusCounter.WithLabelValues(key, emsl.endpoint).Inc()
		eventsCounter.WithLabelValues("Failures", event.Action, event.artifact(), emsl.endpoint).Inc()
		}

		@@ -190,13 +191,13 @@ func (eqc endpointMetricsEventQueueListener) ingress(event Event) {
		eqc.pending.Add(1)

		eventsCounter.WithLabelValues("Events", event.Action, event.artifact(), eqc.endpoint).Inc()
		pendingGauge.Inc()
		pendingGauge.WithLabelValues(eqc.endpoint).Inc()
		}

		func (eqc endpointMetricsEventQueueListener) egress(_ Event) {
		eqc.pending.Add(-1)

		pendingGauge.Dec()
		pendingGauge.WithLabelValues(eqc.endpoint).Dec()
		}

		// endpoints is global registry of endpoints used to report metrics to expvar

notifications/metrics_test.go

+59 −20

Original line number	Diff line number	Diff line
		@@ -33,7 +33,7 @@ func TestMetricsExpvar(t *testing.T) {
		require.NoError(t, err, "unexpected error unmarshaling endpoints")

		if slice, ok := v.([]any); !ok \|\| len(slice) != 1 {
		t.Logf("expected one-element []interface{}, got %#v", v)
		t.Logf("expected one-element []any, got %#v", v)
		}
		}

		@@ -60,16 +60,16 @@ func TestNotificationsMetrics(t *testing.T) {
		eventsCounter.WithLabelValues("Events", "pull", "manifest", "webhook-endpoint").Inc()
		eventsCounter.WithLabelValues("Events", "push", "blob", "webhook-endpoint").Inc()

		// Simulate pending gauge
		pendingGauge.Inc()
		pendingGauge.Inc()
		pendingGauge.Dec()
		// Simulate pending gauge - now includes endpoint label
		pendingGauge.WithLabelValues("webhook-endpoint").Inc()
		pendingGauge.WithLabelValues("webhook-endpoint").Inc()
		pendingGauge.WithLabelValues("webhook-endpoint").Dec()

		// Simulate status counter
		statusCounter.WithLabelValues("200 OK").Inc()
		statusCounter.WithLabelValues("200 OK").Inc()
		statusCounter.WithLabelValues("404 Not Found").Inc()
		statusCounter.WithLabelValues("500 Internal Server Error").Inc()
		// Simulate status counter - now includes endpoint label
		statusCounter.WithLabelValues("200 OK", "webhook-endpoint").Inc()
		statusCounter.WithLabelValues("200 OK", "webhook-endpoint").Inc()
		statusCounter.WithLabelValues("404 Not Found", "webhook-endpoint").Inc()
		statusCounter.WithLabelValues("500 Internal Server Error", "webhook-endpoint").Inc()

		// Simulate error counter
		errorCounter.WithLabelValues("webhook-endpoint").Inc()
		@@ -89,12 +89,12 @@ registry_notifications_events{action="push",artifact="blob",endpoint="webhook-en
		registry_notifications_events{action="push",artifact="manifest",endpoint="webhook-endpoint",type="Events"} 1
		# HELP registry_notifications_pending The gauge of pending events in queue
		# TYPE registry_notifications_pending gauge
		registry_notifications_pending 1
		registry_notifications_pending{endpoint="webhook-endpoint"} 1
		# HELP registry_notifications_status The number of status code
		# TYPE registry_notifications_status counter
		registry_notifications_status{code="200 OK"} 2
		registry_notifications_status{code="404 Not Found"} 1
		registry_notifications_status{code="500 Internal Server Error"} 1
		registry_notifications_status{code="200 OK",endpoint="webhook-endpoint"} 2
		registry_notifications_status{code="404 Not Found",endpoint="webhook-endpoint"} 1
		registry_notifications_status{code="500 Internal Server Error",endpoint="webhook-endpoint"} 1
		`)
		require.NoError(t, err)

		@@ -174,10 +174,10 @@ registry_notifications_events{action="push",artifact="manifest",endpoint="test-e
		registry_notifications_events{action="push",artifact="manifest",endpoint="test-endpoint",type="Successes"} 2
		# HELP registry_notifications_status The number of status code
		# TYPE registry_notifications_status counter
		registry_notifications_status{code="200 OK"} 1
		registry_notifications_status{code="201 Created"} 1
		registry_notifications_status{code="404 Not Found"} 1
		registry_notifications_status{code="500 Internal Server Error"} 1
		registry_notifications_status{code="200 OK",endpoint="test-endpoint"} 1
		registry_notifications_status{code="201 Created",endpoint="test-endpoint"} 1
		registry_notifications_status{code="404 Not Found",endpoint="test-endpoint"} 1
		registry_notifications_status{code="500 Internal Server Error",endpoint="test-endpoint"} 1
		`)
		require.NoError(t, err)

		@@ -249,7 +249,7 @@ registry_notifications_events{action="pull",artifact="blob",endpoint="queue-endp
		registry_notifications_events{action="push",artifact="manifest",endpoint="queue-endpoint",type="Events"} 2
		# HELP registry_notifications_pending The gauge of pending events in queue
		# TYPE registry_notifications_pending gauge
		registry_notifications_pending 1
		registry_notifications_pending{endpoint="queue-endpoint"} 1
		`)
		require.NoError(t, err)

		@@ -360,7 +360,14 @@ func TestSafeMetricsInitialization(t *testing.T) {
		require.Zero(t, sm.failures.Load())
		require.Zero(t, sm.errors.Load())
		require.NotNil(t, sm.statuses)
		require.Empty(t, syncMapToPlainMap(sm.statuses))

		// Check that statuses map is empty
		count := 0
		sm.statuses.Range(func(_, _ any) bool {
		count++
		return true
		})
		require.Zero(t, count, "statuses map should be empty")
		}

		func TestHTTPStatusCodes(t *testing.T) {
		@@ -417,3 +424,35 @@ func TestHTTPStatusCodes(t *testing.T) {
		require.Equal(t, int64(1), v.(*atomic.Int64).Load(), "Status %s should have count 1", key)
		}
		}

		func TestPendingGaugeOperations(t *testing.T) {
		// Create a new registry for isolated testing
		registry := prometheus.NewRegistry()
		registerMetrics(registry)

		endpoint := "gauge-test-endpoint"

		// Test Inc method
		pendingGauge.WithLabelValues(endpoint).Inc()
		pendingGauge.WithLabelValues(endpoint).Inc()
		pendingGauge.WithLabelValues(endpoint).Inc()

		// Test Dec method
		pendingGauge.WithLabelValues(endpoint).Dec()

		// Verify the gauge value
		var expected bytes.Buffer
		_, err := expected.WriteString(`
		# HELP registry_notifications_pending The gauge of pending events in queue
		# TYPE registry_notifications_pending gauge
		registry_notifications_pending{endpoint="gauge-test-endpoint"} 2
		`)
		require.NoError(t, err)

		names := []string{
		fmt.Sprintf("%s_%s_%s", metrics.NamespacePrefix, subsystem, pendingGaugeName),
		}

		err = testutil.GatherAndCompare(registry, &expected, names...)
		require.NoError(t, err)
		}