🐛 Route catalogd content to its leader and webhooks to all replicas

cmd/catalogd/main.go

@@ -92,6 +92,8 @@ type config struct {
 	gcInterval           time.Duration
 	certFile             string
 	keyFile              string
+	webhookCertFile      string
+	webhookKeyFile       string
 	webhookPort          int
 	pullCasDir           string
 	globalPullSecret     string
@@ -133,6 +135,8 @@ func init() {
 	flags.DurationVar(&cfg.gcInterval, "gc-interval", 12*time.Hour, "Garbage collection interval")
 	flags.StringVar(&cfg.certFile, "tls-cert", "", "Certificate file for TLS")
 	flags.StringVar(&cfg.keyFile, "tls-key", "", "Key file for TLS")
+	flags.StringVar(&cfg.webhookCertFile, "webhook-tls-cert", "", "Certificate file for the webhook server TLS (defaults to tls-cert)")
+	flags.StringVar(&cfg.webhookKeyFile, "webhook-tls-key", "", "Key file for the webhook server TLS (defaults to tls-key)")
 	flags.IntVar(&cfg.webhookPort, "webhook-server-port", 9443, "Webhook server port")
 	flags.StringVar(&cfg.pullCasDir, "pull-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to image registries.")
 	flags.StringVar(&cfg.globalPullSecret, "global-pull-secret", "", "Global pull secret (<namespace>/<name>)")
@@ -165,6 +169,16 @@ func validateConfig(cfg *config) error {
 			"certFile", cfg.certFile, "keyFile", cfg.keyFile)
 		return err
 	}
+	if (cfg.webhookCertFile != "" && cfg.webhookKeyFile == "") || (cfg.webhookCertFile == "" && cfg.webhookKeyFile != "") {
+		err := fmt.Errorf("webhook-tls-cert and webhook-tls-key flags must be used together")
+		setupLog.Error(err, "missing webhook TLS configuration",
+			"webhookCertFile", cfg.webhookCertFile, "webhookKeyFile", cfg.webhookKeyFile)
+		return err
+	}
+	if cfg.webhookCertFile == "" && cfg.webhookKeyFile == "" {
+		cfg.webhookCertFile = cfg.certFile
+		cfg.webhookKeyFile = cfg.keyFile
+	}
 
 	if cfg.metricsAddr != "" && cfg.certFile == "" && cfg.keyFile == "" {
 		err := fmt.Errorf("metrics-bind-address requires tls-cert and tls-key flags")
@@ -208,6 +222,14 @@ func run(ctx context.Context) error {
 		setupLog.Error(err, "failed to initialize certificate watcher")
 		return err
 	}
+	webhookCW := cw
+	if cfg.webhookCertFile != cfg.certFile || cfg.webhookKeyFile != cfg.keyFile {
+		webhookCW, err = certwatcher.New(cfg.webhookCertFile, cfg.webhookKeyFile)
+		if err != nil {
+			setupLog.Error(err, "failed to initialize webhook certificate watcher")
+			return err
+		}
+	}
 
 	tlsOpts := func(config *tls.Config) {
 		config.GetCertificate = cw.GetCertificate
@@ -219,6 +241,10 @@ func run(ctx context.Context) error {
 		// For details, see: https://github.com/kubernetes/kubernetes/issues/121197
 		config.NextProtos = []string{"http/1.1"}
 	}
+	webhookTLSOpts := func(config *tls.Config) {
+		config.GetCertificate = webhookCW.GetCertificate
+		config.NextProtos = []string{"http/1.1"}
+	}
 	tlsProfile, err := tlsprofiles.GetTLSConfigFunc()
 	if err != nil {
 		setupLog.Error(err, "failed to get TLS profile")
@@ -229,7 +255,7 @@ func run(ctx context.Context) error {
 	webhookServer := crwebhook.NewServer(crwebhook.Options{
 		Port: cfg.webhookPort,
 		TLSOpts: []func(*tls.Config){
-			tlsOpts,
+			webhookTLSOpts,
 			tlsProfile,
 		},
 	})
@@ -302,6 +328,13 @@ func run(ctx context.Context) error {
 		setupLog.Error(err, "unable to add certificate watcher to manager")
 		return err
 	}
+	if webhookCW != cw {
+		err = mgr.Add(webhookCW)
+		if err != nil {
+			setupLog.Error(err, "unable to add webhook certificate watcher to manager")
+			return err
+		}
+	}
 
 	// This watches the pullCasDir and the SSL_CERT_DIR, and SSL_CERT_FILE for changes
 	cpwPull, err := httputil.NewCertPoolWatcher(cfg.pullCasDir, ctrl.Log.WithName("pull-ca-pool"))
@@ -393,6 +426,8 @@ func run(ctx context.Context) error {
 		CertFile:     cfg.certFile,
 		KeyFile:      cfg.keyFile,
 		LocalStorage: localStorage,
+		PodName:      os.Getenv("POD_NAME"),
+		PodNamespace: os.Getenv("POD_NAMESPACE"),
 		TLSOpts:      []func(*tls.Config){tlsOpts, tlsProfile},
 	}
 

cmd/catalogd/main_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateConfigWebhookTLS(t *testing.T) {
+	t.Run("defaults to the catalog server certificate", func(t *testing.T) {
+		config := &config{
+			certFile: "/var/certs/tls.crt",
+			keyFile:  "/var/certs/tls.key",
+		}
+
+		require.NoError(t, validateConfig(config))
+		require.Equal(t, config.certFile, config.webhookCertFile)
+		require.Equal(t, config.keyFile, config.webhookKeyFile)
+	})
+
+	t.Run("accepts a separate webhook certificate", func(t *testing.T) {
+		config := &config{
+			certFile:        "/var/certs/tls.crt",
+			keyFile:         "/var/certs/tls.key",
+			webhookCertFile: "/var/webhook-certs/tls.crt",
+			webhookKeyFile:  "/var/webhook-certs/tls.key",
+		}
+
+		require.NoError(t, validateConfig(config))
+	})
+
+	t.Run("requires both webhook certificate files", func(t *testing.T) {
+		config := &config{webhookCertFile: "/var/webhook-certs/tls.crt"}
+
+		require.EqualError(t, validateConfig(config), "webhook-tls-cert and webhook-tls-key flags must be used together")
+	})
+}

helm/olmv1/templates/cert-manager/certificate-olmv1-system-catalogd-service-cert.yml

@@ -14,6 +14,8 @@ spec:
     - localhost
     - catalogd-service.{{ .Values.namespaces.olmv1.name }}.svc
     - catalogd-service.{{ .Values.namespaces.olmv1.name }}.svc.cluster.local
+    - catalogd-webhook-service.{{ .Values.namespaces.olmv1.name }}.svc
+    - catalogd-webhook-service.{{ .Values.namespaces.olmv1.name }}.svc.cluster.local
   issuerRef:
     group: cert-manager.io
     kind: ClusterIssuer

helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml

@@ -60,10 +60,14 @@ spec:
             {{- if .Values.options.certManager.enabled }}
             - --tls-cert=/var/certs/tls.crt
             - --tls-key=/var/certs/tls.key
+            - --webhook-tls-cert=/var/certs/tls.crt
+            - --webhook-tls-key=/var/certs/tls.key
             - --pull-cas-dir=/var/ca-certs
             {{- else if .Values.options.openshift.enabled }}
             - --tls-cert=/var/certs/tls.crt
             - --tls-key=/var/certs/tls.key
+            - --webhook-tls-cert=/var/webhook-certs/tls.crt
+            - --webhook-tls-key=/var/webhook-certs/tls.key
             - --v=${LOG_VERBOSITY}
             - --global-pull-secret=openshift-config/pull-secret
             {{- end }}
@@ -76,8 +80,15 @@ spec:
             {{- end }}
           command:
             - ./catalogd
-          {{- if or .Values.options.e2e.enabled .Values.options.openshift.enabled }}
           env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           {{- if .Values.options.e2e.enabled }}
             - name: GOCOVERDIR
               value: /e2e-coverage
@@ -86,7 +97,6 @@ spec:
             - name: SSL_CERT_DIR
               value: /var/ca-certs
           {{- end }}
-          {{- end }}
           image: "{{ .Values.options.catalogd.deployment.image }}"
           name: manager
           {{- if not .Values.options.tilt.enabled }}
@@ -125,6 +135,8 @@ spec:
             {{- else if .Values.options.openshift.enabled }}
             - mountPath: /var/certs
               name: catalogserver-certs
+            - mountPath: /var/webhook-certs
+              name: catalogd-webhook-certs
             - mountPath: /var/ca-certs
               name: ca-certs
               readOnly: true
@@ -176,6 +188,15 @@ spec:
                 path: tls.key
             optional: false
             secretName: catalogserver-cert
+        - name: catalogd-webhook-certs
+          secret:
+            items:
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.key
+            optional: false
+            secretName: catalogd-webhook-service-cert
         - name: ca-certs
           projected:
             sources:

helm/olmv1/templates/mutatingwebhookconfiguration-catalogd-mutating-webhook-configuration.yml

@@ -19,7 +19,7 @@ webhooks:
       - v1
     clientConfig:
       service:
-        name: catalogd-service
+        name: catalogd-webhook-service
         namespace: {{ .Values.namespaces.olmv1.name }}
         path: /mutate-olm-operatorframework-io-v1-clustercatalog
         port: 9443

helm/olmv1/templates/rbac/role-olmv1-system-catalogd-manager-role.yml

@@ -19,4 +19,10 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - patch
 {{- end }}

helm/olmv1/templates/service-olmv1-system-catalogd-service.yml

@@ -18,14 +18,11 @@ spec:
       port: 443
       protocol: TCP
       targetPort: 8443
-    - name: webhook
-      port: 9443
-      protocol: TCP
-      targetPort: 9443
     - name: metrics
       port: 7443
       protocol: TCP
       targetPort: 7443
   selector:
     app.kubernetes.io/name: catalogd
+    olm.operatorframework.io/catalogd-leader: "true"
 {{- end }}

helm/olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml

@@ -0,0 +1,23 @@
+{{- if .Values.options.catalogd.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    {{- include "olmv1.annotations" . | nindent 4 }}
+    {{- if .Values.options.openshift.enabled }}
+    service.beta.openshift.io/serving-cert-secret-name: catalogd-webhook-service-cert
+    {{- end }}
+  labels:
+    app.kubernetes.io/name: catalogd
+    {{- include "olmv1.labels" . | nindent 4 }}
+  name: catalogd-webhook-service
+  namespace: {{ .Values.namespaces.olmv1.name }}
+spec:
+  ports:
+    - name: webhook
+      port: 9443
+      protocol: TCP
+      targetPort: 9443
+  selector:
+    app.kubernetes.io/name: catalogd
+{{- end }}

internal/catalogd/controllers/core/clustercatalog_controller.go

@@ -257,6 +257,9 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *ocv1.
 	if err != nil {
 		unpackErr := fmt.Errorf("source catalog content: %w", err)
 		updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), unpackErr)
+		if !r.Storage.ContentExists(catalog.Name) {
+			updateStatusNotServing(&catalog.Status, catalog.GetGeneration())
+		}
 		return ctrl.Result{}, unpackErr
 	}
 
@@ -266,6 +269,9 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *ocv1.
 	if err := r.Storage.Store(ctx, catalog.Name, fsys); err != nil {
 		storageErr := fmt.Errorf("error storing fbc: %v", err)
 		updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), storageErr)
+		if !r.Storage.ContentExists(catalog.Name) {
+			updateStatusNotServing(&catalog.Status, catalog.GetGeneration())
+		}
 		return ctrl.Result{}, storageErr
 	}
 	baseURL := r.Storage.BaseURL(catalog.Name)

internal/catalogd/controllers/core/clustercatalog_controller_test.go

@@ -27,6 +27,10 @@ import (
 )
 
 func newMockStore(ctrl *gomock.Controller, shouldError bool) *mockstorage.MockInstance {
+	return newMockStoreWithContent(ctrl, shouldError, true)
+}
+
+func newMockStoreWithContent(ctrl *gomock.Controller, shouldError, contentExists bool) *mockstorage.MockInstance {
 	m := mockstorage.NewMockInstance(ctrl)
 	if shouldError {
 		m.EXPECT().Store(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("mockstore store error")).AnyTimes()
@@ -36,10 +40,71 @@ func newMockStore(ctrl *gomock.Controller, shouldError bool) *mockstorage.MockIn
 		m.EXPECT().Delete(gomock.Any()).Return(nil).AnyTimes()
 	}
 	m.EXPECT().BaseURL(gomock.Any()).Return("URL").AnyTimes()
-	m.EXPECT().ContentExists(gomock.Any()).Return(true).AnyTimes()
+	m.EXPECT().ContentExists(gomock.Any()).Return(contentExists).AnyTimes()
 	return m
 }
 
+func TestCatalogdControllerReconcileClearsServingWhenContentIsMissing(t *testing.T) {
+	ref := mustRef(t, "my.org/someimage@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	for name, tc := range map[string]struct {
+		puller     imageutil.Puller
+		storeError bool
+	}{
+		"image pull fails": {
+			puller: &imageutil.FakePuller{Error: errors.New("mock pull error")},
+		},
+		"storage fails": {
+			puller:     &imageutil.FakePuller{ImageFS: &fstest.MapFS{}, Ref: ref},
+			storeError: true,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			mockCtrl := gomock.NewController(t)
+			catalog := &ocv1.ClusterCatalog{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       "catalog",
+					Generation: 2,
+					Finalizers: []string{fbcDeletionFinalizer},
+				},
+				Spec: ocv1.ClusterCatalogSpec{
+					Source: ocv1.CatalogSource{
+						Type:  ocv1.SourceTypeImage,
+						Image: &ocv1.ImageSource{Ref: "my.org/someimage:latest"},
+					},
+				},
+				Status: ocv1.ClusterCatalogStatus{
+					URLs:         &ocv1.ClusterCatalogURLs{Base: "URL"},
+					LastUnpacked: ptr.To(metav1.Now()),
+					ResolvedSource: &ocv1.ResolvedCatalogSource{
+						Type:  ocv1.SourceTypeImage,
+						Image: &ocv1.ResolvedImageSource{Ref: ref.String()},
+					},
+					Conditions: []metav1.Condition{
+						{Type: ocv1.TypeServing, Status: metav1.ConditionTrue, Reason: ocv1.ReasonAvailable},
+					},
+				},
+			}
+			reconciler := &ClusterCatalogReconciler{
+				ImagePuller:    tc.puller,
+				ImageCache:     &imageutil.FakeCache{},
+				Storage:        newMockStoreWithContent(mockCtrl, tc.storeError, false),
+				storedCatalogs: map[string]storedCatalogData{},
+			}
+			require.NoError(t, reconciler.setupFinalizers())
+
+			_, err := reconciler.reconcile(context.Background(), catalog)
+			require.Error(t, err)
+			require.Nil(t, catalog.Status.URLs)
+			require.Nil(t, catalog.Status.LastUnpacked)
+			require.Nil(t, catalog.Status.ResolvedSource)
+			serving := meta.FindStatusCondition(catalog.Status.Conditions, ocv1.TypeServing)
+			require.NotNil(t, serving)
+			assert.Equal(t, metav1.ConditionFalse, serving.Status)
+			assert.Equal(t, ocv1.ReasonUnavailable, serving.Reason)
+		})
+	}
+}
+
 func TestCatalogdControllerReconcile(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	for _, tt := range []struct {