Report the ledger as metrics, with an optional ServiceMonitor and alerts

README.md

@@ -124,6 +124,24 @@ The controller image is published as
 `ghcr.io/cozystack/address-controller:main` (plus `main-<sha>` and semver tags)
 by the release workflow on every push to main.
 
+
+## Metrics
+
+With `metrics.enabled`, the controller serves metrics over HTTPS on port
+8443 behind the Service `address-controller-metrics`. Scrapes are
+authenticated and authorized against the API server, so the scraper's
+identity needs the `address-controller-metrics-reader` ClusterRole. Besides
+the controller-runtime metrics, two gauges count the ledger:
+
+| metric | labels |
+|---|---|
+| `address_controller_ipaddresses` | `class`, `phase` |
+| `address_controller_ipaddressclaims` | `namespace`, `class` (resolved), `phase` |
+
+`metrics.serviceMonitor.enabled` and `metrics.prometheusRule.enabled` add a
+`ServiceMonitor` and alerts where the `monitoring.coreos.com` API exists:
+addresses in `Conflict` or `Lost` for 5 minutes, and claims `Pending` for
+15 minutes.
 ## Development
 
 ```sh

chart/address-controller/templates/controller.yaml

@@ -58,8 +58,13 @@ spec:
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           args:
             - --leader-elect={{ .Values.controller.leaderElect }}
-            - --metrics-bind-address={{ .Values.controller.metricsBindAddress }}
+            - --metrics-bind-address={{ if .Values.metrics.enabled }}:8443{{ else }}{{ .Values.controller.metricsBindAddress }}{{ end }}
             - --health-probe-bind-address=:8081
+          {{- if .Values.metrics.enabled }}
+          ports:
+            - name: metrics
+              containerPort: 8443
+          {{- end }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

chart/address-controller/templates/metrics.yaml

@@ -0,0 +1,118 @@
+{{- if .Values.metrics.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: address-controller-metrics
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: address-controller-metrics
+    {{- include "address-controller.labels" . | nindent 4 }}
+spec:
+  selector:
+    app: address-controller
+  ports:
+    - name: metrics
+      port: 8443
+      targetPort: metrics
+---
+# Lets the controller authenticate and authorize scrape requests.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: address-controller-metrics-auth
+  labels:
+    {{- include "address-controller.labels" . | nindent 4 }}
+rules:
+  - apiGroups: ["authentication.k8s.io"]
+    resources: ["tokenreviews"]
+    verbs: ["create"]
+  - apiGroups: ["authorization.k8s.io"]
+    resources: ["subjectaccessreviews"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: address-controller-metrics-auth
+  labels:
+    {{- include "address-controller.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: address-controller-metrics-auth
+subjects:
+  - kind: ServiceAccount
+    name: address-controller
+    namespace: {{ .Release.Namespace }}
+---
+# Bind a scraper's identity to this role to let it read the metrics.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: address-controller-metrics-reader
+  labels:
+    {{- include "address-controller.labels" . | nindent 4 }}
+rules:
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+{{- if and .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: address-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "address-controller.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: address-controller-metrics
+  endpoints:
+    - port: metrics
+      scheme: https
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      tlsConfig:
+        # The controller serves a self-signed certificate.
+        insecureSkipVerify: true
+{{- end }}
+{{- if and .Values.metrics.prometheusRule.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: address-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "address-controller.labels" . | nindent 4 }}
+spec:
+  groups:
+    - name: address-controller
+      rules:
+        - alert: IPAddressConflict
+          expr: sum by (class) (address_controller_ipaddresses{phase="Conflict"}) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} in Conflict"
+            description: A Service holds an address that its binding does not authorize. kubectl get ipaddresses names the offender.
+        - alert: IPAddressLost
+          expr: sum by (class) (address_controller_ipaddresses{phase="Lost"}) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ "{{ $value }}" }} IPAddress(es) of class {{ "{{ $labels.class }}" }} Lost"
+            description: The backend no longer holds a reserved address. Its claim reports Lost until the address is restored or deleted.
+        - alert: IPAddressClaimPending
+          expr: sum by (namespace, class) (address_controller_ipaddressclaims{phase="Pending"}) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ "{{ $value }}" }} IPAddressClaim(s) in {{ "{{ $labels.namespace }}" }} Pending for 15m"
+            description: The claim's class may be exhausted, missing or refused by its driver. kubectl describe on the claim shows why.
+{{- end }}
+{{- end }}

chart/address-controller/values.yaml

@@ -18,3 +18,20 @@ controller:
       memory: 64Mi
     limits:
       memory: 128Mi
+
+metrics:
+  # Serve controller-runtime and inventory metrics (address_controller_*)
+  # over HTTPS on 8443, behind the Service address-controller-metrics.
+  # Requests are authenticated and authorized against the API server, so a
+  # scraper needs GET on the /metrics non-resource URL: bind its identity
+  # to the address-controller-metrics-reader ClusterRole. Overrides
+  # controller.metricsBindAddress.
+  enabled: false
+  serviceMonitor:
+    # A monitoring.coreos.com ServiceMonitor for the Service; rendered only
+    # where that API exists.
+    enabled: false
+  prometheusRule:
+    # Alerts on Conflict and Lost addresses and on long-Pending claims;
+    # rendered only where monitoring.coreos.com exists.
+    enabled: false

cmd/main.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -207,6 +208,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	metrics.Registry.MustRegister(&controller.InventoryCollector{Reader: mgr.GetClient()})
+
 	if err = controller.SetupIndexes(context.Background(), mgr); err != nil {
 		setupLog.Error(err, "unable to set up field indexes")
 		os.Exit(1)

go.mod

@@ -5,6 +5,7 @@ go 1.26
 godebug default=go1.23
 
 require (
+	github.com/prometheus/client_golang v1.19.1
 	k8s.io/api v0.32.0
 	k8s.io/apimachinery v0.32.0
 	k8s.io/client-go v0.32.0
@@ -48,7 +49,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

internal/controller/metrics.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1"
+)
+
+var (
+	addressesDesc = prometheus.NewDesc(
+		"address_controller_ipaddresses",
+		"IPAddress objects by class and phase.",
+		[]string{"class", "phase"}, nil)
+	claimsDesc = prometheus.NewDesc(
+		"address_controller_ipaddressclaims",
+		"IPAddressClaim objects by namespace, resolved class and phase.",
+		[]string{"namespace", "class", "phase"}, nil)
+)
+
+// InventoryCollector reports the ledger as gauges, counted from the cache
+// at scrape time, so the counts hold no state of their own.
+type InventoryCollector struct {
+	Reader client.Reader
+}
+
+// Describe implements prometheus.Collector.
+func (c *InventoryCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- addressesDesc
+	ch <- claimsDesc
+}
+
+// Collect implements prometheus.Collector. A failed list reports an
+// invalid metric, which fails the scrape visibly instead of reporting zero.
+func (c *InventoryCollector) Collect(ch chan<- prometheus.Metric) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	addrs := &localv1alpha1.IPAddressList{}
+	if err := c.Reader.List(ctx, addrs); err != nil {
+		ch <- prometheus.NewInvalidMetric(addressesDesc, err)
+	} else {
+		counts := map[[2]string]int{}
+		for _, a := range addrs.Items {
+			counts[[2]string{a.Spec.ClassName, string(a.Status.Phase)}]++
+		}
+		for k, n := range counts {
+			ch <- prometheus.MustNewConstMetric(addressesDesc, prometheus.GaugeValue, float64(n), k[0], k[1])
+		}
+	}
+
+	claims := &localv1alpha1.IPAddressClaimList{}
+	if err := c.Reader.List(ctx, claims); err != nil {
+		ch <- prometheus.NewInvalidMetric(claimsDesc, err)
+		return
+	}
+	counts := map[[3]string]int{}
+	for _, cl := range claims.Items {
+		counts[[3]string{cl.Namespace, cl.Status.ClassName, string(cl.Status.Phase)}]++
+	}
+	for k, n := range counts {
+		ch <- prometheus.MustNewConstMetric(claimsDesc, prometheus.GaugeValue, float64(n), k[0], k[1], k[2])
+	}
+}

internal/controller/metrics_test.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
+	localv1alpha1 "github.com/cozystack/address-controller/api/v1alpha1"
+)
+
+func TestInventoryCollectorCountsByPhase(t *testing.T) {
+	bound := boundAddress("ip-a", "203.0.113.50", localv1alpha1.IPAddressBound)
+	conflict := boundAddress("ip-b", "203.0.113.51", localv1alpha1.IPAddressConflict)
+	conflict2 := boundAddress("ip-c", "203.0.113.52", localv1alpha1.IPAddressConflict)
+	claim := pendingClaim(testClassName)
+	claim.Status.Phase = localv1alpha1.ClaimBound
+	claim.Status.ClassName = testClassName
+	c := testClient(t, bound, conflict, conflict2, claim)
+
+	want := `
+# HELP address_controller_ipaddresses IPAddress objects by class and phase.
+# TYPE address_controller_ipaddresses gauge
+address_controller_ipaddresses{class="public",phase="Bound"} 1
+address_controller_ipaddresses{class="public",phase="Conflict"} 2
+# HELP address_controller_ipaddressclaims IPAddressClaim objects by namespace, resolved class and phase.
+# TYPE address_controller_ipaddressclaims gauge
+address_controller_ipaddressclaims{class="public",namespace="tenant-a",phase="Bound"} 1
+`
+	if err := testutil.CollectAndCompare(&InventoryCollector{Reader: c}, strings.NewReader(want)); err != nil {
+		t.Error(err)
+	}
+}