From 2dd5815f6c2b4a06632a7ca9364d76a1dce61f3d Mon Sep 17 00:00:00 2001 From: Todd Short Date: Tue, 29 Sep 2026 12:03:30 -0400 Subject: [PATCH 1/3] fix(catalogd): route catalog service to the leader Remove non-leader replicas from the catalog content Service so requests remain available through the elected catalogd leader. Signed-off-by: Todd Short --- cmd/catalogd/main.go | 2 + ...mv1-system-catalogd-controller-manager.yml | 10 +- ...ole-olmv1-system-catalogd-manager-role.yml | 6 + .../service-olmv1-system-catalogd-service.yml | 1 + internal/catalogd/serverutil/serverutil.go | 142 ++++++++++++++++-- .../catalogd/serverutil/serverutil_test.go | 87 +++++++++++ manifests/experimental-e2e.yaml | 15 ++ manifests/experimental.yaml | 16 ++ manifests/standard-e2e.yaml | 15 ++ manifests/standard.yaml | 16 ++ test/e2e/steps/demo_steps.go | 8 +- 11 files changed, 300 insertions(+), 18 deletions(-) diff --git a/cmd/catalogd/main.go b/cmd/catalogd/main.go index 2b6f81b4ab..40e1ab63db 100644 --- a/cmd/catalogd/main.go +++ b/cmd/catalogd/main.go @@ -393,6 +393,8 @@ func run(ctx context.Context) error { CertFile: cfg.certFile, KeyFile: cfg.keyFile, LocalStorage: localStorage, + PodName: os.Getenv("POD_NAME"), + PodNamespace: os.Getenv("POD_NAMESPACE"), TLSOpts: []func(*tls.Config){tlsOpts, tlsProfile}, } diff --git a/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml b/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml index ac69bce6a4..f49e38ae9f 100644 --- a/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml +++ b/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml @@ -76,8 +76,15 @@ spec: {{- end }} command: - ./catalogd - {{- if or .Values.options.e2e.enabled .Values.options.openshift.enabled }} env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace {{- if .Values.options.e2e.enabled }} - name: GOCOVERDIR value: /e2e-coverage @@ -86,7 +93,6 @@ spec: - name: SSL_CERT_DIR value: /var/ca-certs {{- end }} - {{- end }} image: "{{ .Values.options.catalogd.deployment.image }}" name: manager {{- if not .Values.options.tilt.enabled }} diff --git a/helm/olmv1/templates/rbac/role-olmv1-system-catalogd-manager-role.yml b/helm/olmv1/templates/rbac/role-olmv1-system-catalogd-manager-role.yml index 09cec7c0c2..cfa74a1960 100644 --- a/helm/olmv1/templates/rbac/role-olmv1-system-catalogd-manager-role.yml +++ b/helm/olmv1/templates/rbac/role-olmv1-system-catalogd-manager-role.yml @@ -19,4 +19,10 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - patch {{- end }} diff --git a/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml b/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml index eca9593995..a9f0bedb29 100644 --- a/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml +++ b/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml @@ -28,4 +28,5 @@ spec: targetPort: 7443 selector: app.kubernetes.io/name: catalogd + olm.operatorframework.io/catalogd-leader: "true" {{- end }} diff --git a/internal/catalogd/serverutil/serverutil.go b/internal/catalogd/serverutil/serverutil.go index 3a597de877..97b228144e 100644 --- a/internal/catalogd/serverutil/serverutil.go +++ b/internal/catalogd/serverutil/serverutil.go @@ -13,19 +13,30 @@ import ( "github.com/go-logr/logr" "github.com/gorilla/handlers" "github.com/klauspost/compress/gzhttp" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" + ocv1 "github.com/operator-framework/operator-controller/api/v1" catalogdmetrics "github.com/operator-framework/operator-controller/internal/catalogd/metrics" "github.com/operator-framework/operator-controller/internal/catalogd/storage" ) +const catalogdLeaderLabel = "olm.operatorframework.io/catalogd-leader" + type CatalogServerConfig struct { ExternalAddr string CatalogAddr string CertFile string KeyFile string LocalStorage storage.Instance + PodName string + PodNamespace string // TLSOpts are optional functions applied to the TLS configuration when serving over HTTPS. // Use these to configure cipher suites, minimum TLS version, curve preferences, and // certificate retrieval (e.g. via a certwatcher). @@ -33,11 +44,14 @@ type CatalogServerConfig struct { } // AddCatalogServerToManager adds the catalog HTTP server to the manager and registers -// a readiness check that passes once the server has started serving. Because -// NeedLeaderElection returns false, Start() is called on every pod immediately, so all -// replicas bind the catalog port and become ready. Non-leader pods serve requests but -// return 404 (empty local cache); callers are expected to retry. +// a readiness check that passes once the server has started serving. Because +// NeedLeaderElection returns false, every replica binds the catalog port and becomes +// ready. catalogd-service selects only the leader-labelled replica. func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig) error { + if cfg.PodName == "" || cfg.PodNamespace == "" { + return errors.New("catalog server requires the catalogd pod name and namespace") + } + shutdownTimeout := 30 * time.Second r := &catalogServerRunnable{ cfg: cfg, @@ -55,10 +69,25 @@ func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig) error return fmt.Errorf("error adding catalog server to manager: %w", err) } + // The Service selects this label, so only the elected pod is an endpoint. + // This runnable itself is not leader-gated: it first removes a potentially + // stale label on every pod, then waits for this manager to become leader. + leaderLabeler := &catalogdLeaderLabeler{ + client: mgr.GetClient(), + storage: cfg.LocalStorage, + electedCh: mgr.Elected(), + pod: types.NamespacedName{ + Name: cfg.PodName, + Namespace: cfg.PodNamespace, + }, + } + if err := mgr.Add(leaderLabeler); err != nil { + return fmt.Errorf("error adding catalog leader labeler to manager: %w", err) + } + // Register a readiness check that passes once Start() has been called and the - // server is actively serving. All pods reach Start() (NeedLeaderElection=false), - // so all replicas become ready and receive traffic; non-leaders return 404 until - // they win the leader lease and populate their local cache. + // server is actively serving. All pods reach Start() (NeedLeaderElection=false) + // so rolling updates do not wait for a replacement pod to win leadership. if err := mgr.AddReadyzCheck("catalog-server", r.readyzCheck()); err != nil { return fmt.Errorf("error adding catalog server readiness check: %w", err) } @@ -67,8 +96,7 @@ func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig) error } // catalogServerRunnable is a Runnable that binds the catalog HTTP port on every pod. -// Because NeedLeaderElection returns false, Start() is called on all replicas immediately; -// non-leader pods serve the catalog port but return 404 (empty local cache). +// Because NeedLeaderElection returns false, Start() is called on all replicas immediately. type catalogServerRunnable struct { cfg CatalogServerConfig server *http.Server @@ -83,9 +111,8 @@ type catalogServerRunnable struct { // (held by the still-running old pod) and therefore could never pass the // catalog-server readiness check, deadlocking the rollout. // -// Non-leader pods serve the catalog HTTP port but have an empty local cache -// (only the leader's reconciler downloads catalog content), so requests to a -// non-leader return 404. Callers are expected to retry. +// Non-leader pods serve the catalog HTTP port but are excluded from catalogd-service +// by the leader label because only the leader's reconciler downloads catalog content. func (r *catalogServerRunnable) NeedLeaderElection() bool { return false } func (r *catalogServerRunnable) Start(ctx context.Context) error { @@ -143,6 +170,97 @@ func (r *catalogServerRunnable) readyzCheck() healthz.Checker { } } +// catalogdLeaderLabeler manages the label selected by catalogd-service. It starts +// on every replica so a pod that restarts after losing leadership cannot retain a +// stale leader label. It only adds the label after it has become leader and has a +// local copy of every catalog that is currently advertised as serving. +type catalogdLeaderLabeler struct { + client client.Client + storage storage.Instance + electedCh <-chan struct{} + pod types.NamespacedName +} + +// NeedLeaderElection returns false so the label is removed from every replica +// before it can receive traffic. Start waits for the manager's Elected channel +// before adding it back to the elected replica. +func (r *catalogdLeaderLabeler) NeedLeaderElection() bool { return false } + +func (r *catalogdLeaderLabeler) Start(ctx context.Context) error { + if err := r.setLeaderLabel(ctx, false); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("removing catalogd leader label from pod %s: %w", r.pod, err) + } + + select { + case <-ctx.Done(): + return nil + case <-r.electedCh: + } + + if err := r.waitForCatalogContent(ctx); err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + return err + } + + if err := r.setLeaderLabel(ctx, true); err != nil { + return fmt.Errorf("adding catalogd leader label to pod %s: %w", r.pod, err) + } + + <-ctx.Done() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := r.setLeaderLabel(cleanupCtx, false); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("removing catalogd leader label from pod %s during shutdown: %w", r.pod, err) + } + return nil +} + +func (r *catalogdLeaderLabeler) waitForCatalogContent(ctx context.Context) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + ready, err := r.catalogContentAvailable(ctx) + if err != nil { + return err + } + if ready { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func (r *catalogdLeaderLabeler) catalogContentAvailable(ctx context.Context) (bool, error) { + var catalogs ocv1.ClusterCatalogList + if err := r.client.List(ctx, &catalogs); err != nil { + return false, fmt.Errorf("listing ClusterCatalogs while waiting to serve content: %w", err) + } + for _, catalog := range catalogs.Items { + if meta.IsStatusConditionPresentAndEqual(catalog.Status.Conditions, ocv1.TypeServing, metav1.ConditionTrue) && !r.storage.ContentExists(catalog.Name) { + return false, nil + } + } + return true, nil +} + +func (r *catalogdLeaderLabeler) setLeaderLabel(ctx context.Context, leader bool) error { + value := "null" + if leader { + value = `"true"` + } + patch := []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":%s}}}`, catalogdLeaderLabel, value)) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: r.pod.Name, Namespace: r.pod.Namespace}} + return r.client.Patch(ctx, pod, client.RawPatch(types.MergePatchType, patch)) +} + func logrLoggingHandler(l logr.Logger, handler http.Handler) http.Handler { return handlers.CustomLoggingHandler(nil, handler, func(_ io.Writer, params handlers.LogFormatterParams) { username := "-" diff --git a/internal/catalogd/serverutil/serverutil_test.go b/internal/catalogd/serverutil/serverutil_test.go index 57ebf94f82..36f50f3e93 100644 --- a/internal/catalogd/serverutil/serverutil_test.go +++ b/internal/catalogd/serverutil/serverutil_test.go @@ -10,6 +10,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "io" "math/big" "net" @@ -23,7 +24,13 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + ocv1 "github.com/operator-framework/operator-controller/api/v1" mockstorage "github.com/operator-framework/operator-controller/internal/testutil/mock/storage" ) @@ -260,3 +267,83 @@ func TestCatalogServerTLSOptsCertSourceRequired(t *testing.T) { err := r.Start(ctx) require.ErrorContains(t, err, "TLSOpts must configure a certificate source") } + +func TestCatalogdLeaderLabelerSetLeaderLabel(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "catalogd-0", + Namespace: "olmv1-system", + Labels: map[string]string{"app.kubernetes.io/name": "catalogd"}, + }} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pod).Build() + r := &catalogdLeaderLabeler{ + client: c, + pod: types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, + } + + require.NoError(t, r.setLeaderLabel(context.Background(), true)) + require.NoError(t, c.Get(context.Background(), r.pod, pod)) + require.Equal(t, "true", pod.Labels[catalogdLeaderLabel]) + + require.NoError(t, r.setLeaderLabel(context.Background(), false)) + require.NoError(t, c.Get(context.Background(), r.pod, pod)) + _, found := pod.Labels[catalogdLeaderLabel] + require.False(t, found) +} + +func TestCatalogdLeaderLabelerCatalogContentAvailable(t *testing.T) { + tests := []struct { + name string + contentExists bool + expectedReady bool + servingCatalogs int + }{ + { + name: "all advertised catalogs are present", + contentExists: true, + expectedReady: true, + servingCatalogs: 1, + }, + { + name: "advertised catalog is missing", + contentExists: false, + expectedReady: false, + servingCatalogs: 1, + }, + { + name: "no catalog is advertised", + expectedReady: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + store := mockstorage.NewMockInstance(mockCtrl) + + objects := make([]runtime.Object, 0, tt.servingCatalogs) + for i := range tt.servingCatalogs { + name := fmt.Sprintf("catalog-%d", i) + store.EXPECT().ContentExists(name).Return(tt.contentExists) + objects = append(objects, &ocv1.ClusterCatalog{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: ocv1.ClusterCatalogStatus{Conditions: []metav1.Condition{{ + Type: ocv1.TypeServing, + Status: metav1.ConditionTrue, + }}}, + }) + } + + scheme := runtime.NewScheme() + require.NoError(t, ocv1.AddToScheme(scheme)) + r := &catalogdLeaderLabeler{ + client: fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build(), + storage: store, + } + ready, err := r.catalogContentAvailable(context.Background()) + require.NoError(t, err) + require.Equal(t, tt.expectedReady, ready) + }) + } +} diff --git a/manifests/experimental-e2e.yaml b/manifests/experimental-e2e.yaml index 683d3969df..78c20d7b9d 100644 --- a/manifests/experimental-e2e.yaml +++ b/manifests/experimental-e2e.yaml @@ -2272,6 +2272,12 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - patch --- # Source: olmv1/templates/rbac/role-olmv1-system-common-leader-election-role.yml apiVersion: rbac.authorization.k8s.io/v1 @@ -2478,6 +2484,7 @@ spec: targetPort: 7443 selector: app.kubernetes.io/name: catalogd + olm.operatorframework.io/catalogd-leader: "true" --- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 @@ -2591,6 +2598,14 @@ spec: command: - ./catalogd env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: GOCOVERDIR value: /e2e-coverage image: "quay.io/operator-framework/catalogd:devel" diff --git a/manifests/experimental.yaml b/manifests/experimental.yaml index 1d13fa0c87..c57b9397e7 100644 --- a/manifests/experimental.yaml +++ b/manifests/experimental.yaml @@ -2233,6 +2233,12 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - patch --- # Source: olmv1/templates/rbac/role-olmv1-system-common-leader-election-role.yml apiVersion: rbac.authorization.k8s.io/v1 @@ -2439,6 +2445,7 @@ spec: targetPort: 7443 selector: app.kubernetes.io/name: catalogd + olm.operatorframework.io/catalogd-leader: "true" --- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 @@ -2505,6 +2512,15 @@ spec: - --pull-cas-dir=/var/ca-certs command: - ./catalogd + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: "quay.io/operator-framework/catalogd:devel" name: manager livenessProbe: diff --git a/manifests/standard-e2e.yaml b/manifests/standard-e2e.yaml index 28dca6563d..55f46d3262 100644 --- a/manifests/standard-e2e.yaml +++ b/manifests/standard-e2e.yaml @@ -1425,6 +1425,12 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - patch --- # Source: olmv1/templates/rbac/role-olmv1-system-common-leader-election-role.yml apiVersion: rbac.authorization.k8s.io/v1 @@ -1631,6 +1637,7 @@ spec: targetPort: 7443 selector: app.kubernetes.io/name: catalogd + olm.operatorframework.io/catalogd-leader: "true" --- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 @@ -1743,6 +1750,14 @@ spec: command: - ./catalogd env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: GOCOVERDIR value: /e2e-coverage image: "quay.io/operator-framework/catalogd:devel" diff --git a/manifests/standard.yaml b/manifests/standard.yaml index 71c7677772..29a7c6bc76 100644 --- a/manifests/standard.yaml +++ b/manifests/standard.yaml @@ -1386,6 +1386,12 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - patch --- # Source: olmv1/templates/rbac/role-olmv1-system-common-leader-election-role.yml apiVersion: rbac.authorization.k8s.io/v1 @@ -1592,6 +1598,7 @@ spec: targetPort: 7443 selector: app.kubernetes.io/name: catalogd + olm.operatorframework.io/catalogd-leader: "true" --- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 @@ -1657,6 +1664,15 @@ spec: - --pull-cas-dir=/var/ca-certs command: - ./catalogd + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: "quay.io/operator-framework/catalogd:devel" name: manager livenessProbe: diff --git a/test/e2e/steps/demo_steps.go b/test/e2e/steps/demo_steps.go index 80346bff88..2a58964d45 100644 --- a/test/e2e/steps/demo_steps.go +++ b/test/e2e/steps/demo_steps.go @@ -151,9 +151,8 @@ func catalogPortForwardAlive(addr string) bool { } // resetCatalogPortForward tears down the cached port-forward so the next -// call to ensureCatalogPortForward establishes a fresh connection. With -// CatalogdHA, non-leader pods return 404 (empty local cache); resetting -// lets the next retry potentially reach the leader pod. +// call to ensureCatalogPortForward establishes a fresh connection. This also +// recovers a connection interrupted while catalogd elects a new leader. func resetCatalogPortForward(ctx context.Context) { sc := scenarioCtx(ctx) if sc.catalogCleanup != nil { @@ -168,7 +167,8 @@ func catalogCurlJq(ctx context.Context, catalogName, jqFilter string) (string, e if err != nil { return "", err } - // pipefail: propagate curl exit code through the pipe (e.g. HTTP 404 from a non-leader catalogd pod). + // pipefail: propagate curl exit code through the pipe so connection or HTTP errors + // do not reach jq as non-JSON input. // -sS: silent but show errors on stderr. -k: skip TLS verification for the port-forward. // --compressed: request gzip and stream-decompress (catalogd uses gzhttp); saves network for the large operatorhubio catalog. // --fail: exit 22 on HTTP errors so non-JSON error bodies don't reach jq. From fda19c90dc2ef743814001c9729c671cf4bf62c0 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Tue, 29 Sep 2026 13:43:25 -0400 Subject: [PATCH 2/3] fix(catalogd): serve webhooks from every replica Route admission requests through a separate Service that selects every ready catalogd replica while catalog content remains leader-only. Configure TLS for the new Service with cert-manager and OpenShift service certificates. Signed-off-by: Todd Short --- cmd/catalogd/main.go | 35 +++++++++++- cmd/catalogd/main_test.go | 53 +++++++++++++++++++ ...ate-olmv1-system-catalogd-service-cert.yml | 2 + ...mv1-system-catalogd-controller-manager.yml | 15 ++++++ ...atalogd-mutating-webhook-configuration.yml | 2 +- .../service-olmv1-system-catalogd-service.yml | 4 -- ...-olmv1-system-catalogd-webhook-service.yml | 23 ++++++++ manifests/experimental-e2e.yaml | 30 +++++++++-- manifests/experimental.yaml | 30 +++++++++-- manifests/standard-e2e.yaml | 30 +++++++++-- manifests/standard.yaml | 30 +++++++++-- 11 files changed, 228 insertions(+), 26 deletions(-) create mode 100644 cmd/catalogd/main_test.go create mode 100644 helm/olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml diff --git a/cmd/catalogd/main.go b/cmd/catalogd/main.go index 40e1ab63db..fca1563ae4 100644 --- a/cmd/catalogd/main.go +++ b/cmd/catalogd/main.go @@ -92,6 +92,8 @@ type config struct { gcInterval time.Duration certFile string keyFile string + webhookCertFile string + webhookKeyFile string webhookPort int pullCasDir string globalPullSecret string @@ -133,6 +135,8 @@ func init() { flags.DurationVar(&cfg.gcInterval, "gc-interval", 12*time.Hour, "Garbage collection interval") flags.StringVar(&cfg.certFile, "tls-cert", "", "Certificate file for TLS") flags.StringVar(&cfg.keyFile, "tls-key", "", "Key file for TLS") + flags.StringVar(&cfg.webhookCertFile, "webhook-tls-cert", "", "Certificate file for the webhook server TLS (defaults to tls-cert)") + flags.StringVar(&cfg.webhookKeyFile, "webhook-tls-key", "", "Key file for the webhook server TLS (defaults to tls-key)") flags.IntVar(&cfg.webhookPort, "webhook-server-port", 9443, "Webhook server port") flags.StringVar(&cfg.pullCasDir, "pull-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to image registries.") flags.StringVar(&cfg.globalPullSecret, "global-pull-secret", "", "Global pull secret (/)") @@ -165,6 +169,16 @@ func validateConfig(cfg *config) error { "certFile", cfg.certFile, "keyFile", cfg.keyFile) return err } + if (cfg.webhookCertFile != "" && cfg.webhookKeyFile == "") || (cfg.webhookCertFile == "" && cfg.webhookKeyFile != "") { + err := fmt.Errorf("webhook-tls-cert and webhook-tls-key flags must be used together") + setupLog.Error(err, "missing webhook TLS configuration", + "webhookCertFile", cfg.webhookCertFile, "webhookKeyFile", cfg.webhookKeyFile) + return err + } + if cfg.webhookCertFile == "" && cfg.webhookKeyFile == "" { + cfg.webhookCertFile = cfg.certFile + cfg.webhookKeyFile = cfg.keyFile + } if cfg.metricsAddr != "" && cfg.certFile == "" && cfg.keyFile == "" { err := fmt.Errorf("metrics-bind-address requires tls-cert and tls-key flags") @@ -208,6 +222,14 @@ func run(ctx context.Context) error { setupLog.Error(err, "failed to initialize certificate watcher") return err } + webhookCW := cw + if cfg.webhookCertFile != cfg.certFile || cfg.webhookKeyFile != cfg.keyFile { + webhookCW, err = certwatcher.New(cfg.webhookCertFile, cfg.webhookKeyFile) + if err != nil { + setupLog.Error(err, "failed to initialize webhook certificate watcher") + return err + } + } tlsOpts := func(config *tls.Config) { config.GetCertificate = cw.GetCertificate @@ -219,6 +241,10 @@ func run(ctx context.Context) error { // For details, see: https://github.com/kubernetes/kubernetes/issues/121197 config.NextProtos = []string{"http/1.1"} } + webhookTLSOpts := func(config *tls.Config) { + config.GetCertificate = webhookCW.GetCertificate + config.NextProtos = []string{"http/1.1"} + } tlsProfile, err := tlsprofiles.GetTLSConfigFunc() if err != nil { setupLog.Error(err, "failed to get TLS profile") @@ -229,7 +255,7 @@ func run(ctx context.Context) error { webhookServer := crwebhook.NewServer(crwebhook.Options{ Port: cfg.webhookPort, TLSOpts: []func(*tls.Config){ - tlsOpts, + webhookTLSOpts, tlsProfile, }, }) @@ -302,6 +328,13 @@ func run(ctx context.Context) error { setupLog.Error(err, "unable to add certificate watcher to manager") return err } + if webhookCW != cw { + err = mgr.Add(webhookCW) + if err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher to manager") + return err + } + } // This watches the pullCasDir and the SSL_CERT_DIR, and SSL_CERT_FILE for changes cpwPull, err := httputil.NewCertPoolWatcher(cfg.pullCasDir, ctrl.Log.WithName("pull-ca-pool")) diff --git a/cmd/catalogd/main_test.go b/cmd/catalogd/main_test.go new file mode 100644 index 0000000000..11070bd353 --- /dev/null +++ b/cmd/catalogd/main_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateConfigWebhookTLS(t *testing.T) { + t.Run("defaults to the catalog server certificate", func(t *testing.T) { + config := &config{ + certFile: "/var/certs/tls.crt", + keyFile: "/var/certs/tls.key", + } + + require.NoError(t, validateConfig(config)) + require.Equal(t, config.certFile, config.webhookCertFile) + require.Equal(t, config.keyFile, config.webhookKeyFile) + }) + + t.Run("accepts a separate webhook certificate", func(t *testing.T) { + config := &config{ + certFile: "/var/certs/tls.crt", + keyFile: "/var/certs/tls.key", + webhookCertFile: "/var/webhook-certs/tls.crt", + webhookKeyFile: "/var/webhook-certs/tls.key", + } + + require.NoError(t, validateConfig(config)) + }) + + t.Run("requires both webhook certificate files", func(t *testing.T) { + config := &config{webhookCertFile: "/var/webhook-certs/tls.crt"} + + require.EqualError(t, validateConfig(config), "webhook-tls-cert and webhook-tls-key flags must be used together") + }) +} diff --git a/helm/olmv1/templates/cert-manager/certificate-olmv1-system-catalogd-service-cert.yml b/helm/olmv1/templates/cert-manager/certificate-olmv1-system-catalogd-service-cert.yml index 7c6311eedf..8676d039c7 100644 --- a/helm/olmv1/templates/cert-manager/certificate-olmv1-system-catalogd-service-cert.yml +++ b/helm/olmv1/templates/cert-manager/certificate-olmv1-system-catalogd-service-cert.yml @@ -14,6 +14,8 @@ spec: - localhost - catalogd-service.{{ .Values.namespaces.olmv1.name }}.svc - catalogd-service.{{ .Values.namespaces.olmv1.name }}.svc.cluster.local + - catalogd-webhook-service.{{ .Values.namespaces.olmv1.name }}.svc + - catalogd-webhook-service.{{ .Values.namespaces.olmv1.name }}.svc.cluster.local issuerRef: group: cert-manager.io kind: ClusterIssuer diff --git a/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml b/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml index f49e38ae9f..cd0b473fa8 100644 --- a/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml +++ b/helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml @@ -60,10 +60,14 @@ spec: {{- if .Values.options.certManager.enabled }} - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/certs/tls.crt + - --webhook-tls-key=/var/certs/tls.key - --pull-cas-dir=/var/ca-certs {{- else if .Values.options.openshift.enabled }} - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/webhook-certs/tls.crt + - --webhook-tls-key=/var/webhook-certs/tls.key - --v=${LOG_VERBOSITY} - --global-pull-secret=openshift-config/pull-secret {{- end }} @@ -131,6 +135,8 @@ spec: {{- else if .Values.options.openshift.enabled }} - mountPath: /var/certs name: catalogserver-certs + - mountPath: /var/webhook-certs + name: catalogd-webhook-certs - mountPath: /var/ca-certs name: ca-certs readOnly: true @@ -182,6 +188,15 @@ spec: path: tls.key optional: false secretName: catalogserver-cert + - name: catalogd-webhook-certs + secret: + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: catalogd-webhook-service-cert - name: ca-certs projected: sources: diff --git a/helm/olmv1/templates/mutatingwebhookconfiguration-catalogd-mutating-webhook-configuration.yml b/helm/olmv1/templates/mutatingwebhookconfiguration-catalogd-mutating-webhook-configuration.yml index 95077c9ffe..4349127560 100644 --- a/helm/olmv1/templates/mutatingwebhookconfiguration-catalogd-mutating-webhook-configuration.yml +++ b/helm/olmv1/templates/mutatingwebhookconfiguration-catalogd-mutating-webhook-configuration.yml @@ -19,7 +19,7 @@ webhooks: - v1 clientConfig: service: - name: catalogd-service + name: catalogd-webhook-service namespace: {{ .Values.namespaces.olmv1.name }} path: /mutate-olm-operatorframework-io-v1-clustercatalog port: 9443 diff --git a/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml b/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml index a9f0bedb29..a3135ca152 100644 --- a/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml +++ b/helm/olmv1/templates/service-olmv1-system-catalogd-service.yml @@ -18,10 +18,6 @@ spec: port: 443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 - name: metrics port: 7443 protocol: TCP diff --git a/helm/olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml b/helm/olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml new file mode 100644 index 0000000000..914602c24b --- /dev/null +++ b/helm/olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml @@ -0,0 +1,23 @@ +{{- if .Values.options.catalogd.enabled }} +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- include "olmv1.annotations" . | nindent 4 }} + {{- if .Values.options.openshift.enabled }} + service.beta.openshift.io/serving-cert-secret-name: catalogd-webhook-service-cert + {{- end }} + labels: + app.kubernetes.io/name: catalogd + {{- include "olmv1.labels" . | nindent 4 }} + name: catalogd-webhook-service + namespace: {{ .Values.namespaces.olmv1.name }} +spec: + ports: + - name: webhook + port: 9443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/name: catalogd +{{- end }} diff --git a/manifests/experimental-e2e.yaml b/manifests/experimental-e2e.yaml index 78c20d7b9d..d88cf56409 100644 --- a/manifests/experimental-e2e.yaml +++ b/manifests/experimental-e2e.yaml @@ -2474,10 +2474,6 @@ spec: port: 443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 - name: metrics port: 7443 protocol: TCP @@ -2486,6 +2482,26 @@ spec: app.kubernetes.io/name: catalogd olm.operatorframework.io/catalogd-leader: "true" --- +# Source: olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml +apiVersion: v1 +kind: Service +metadata: + annotations: + olm.operatorframework.io/feature-set: experimental-e2e + labels: + app.kubernetes.io/name: catalogd + app.kubernetes.io/part-of: olm + name: catalogd-webhook-service + namespace: olmv1-system +spec: + ports: + - name: webhook + port: 9443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/name: catalogd +--- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 kind: Service @@ -2590,6 +2606,8 @@ spec: - --feature-gates=GraphQLCatalogQueries=true - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/certs/tls.crt + - --webhook-tls-key=/var/certs/tls.key - --pull-cas-dir=/var/ca-certs - --tls-profile=custom - --tls-custom-version=TLSv1.3 @@ -2911,6 +2929,8 @@ spec: - localhost - catalogd-service.olmv1-system.svc - catalogd-service.olmv1-system.svc.cluster.local + - catalogd-webhook-service.olmv1-system.svc + - catalogd-webhook-service.olmv1-system.svc.cluster.local issuerRef: group: cert-manager.io kind: ClusterIssuer @@ -2990,7 +3010,7 @@ webhooks: - v1 clientConfig: service: - name: catalogd-service + name: catalogd-webhook-service namespace: olmv1-system path: /mutate-olm-operatorframework-io-v1-clustercatalog port: 9443 diff --git a/manifests/experimental.yaml b/manifests/experimental.yaml index c57b9397e7..750b0a9715 100644 --- a/manifests/experimental.yaml +++ b/manifests/experimental.yaml @@ -2435,10 +2435,6 @@ spec: port: 443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 - name: metrics port: 7443 protocol: TCP @@ -2447,6 +2443,26 @@ spec: app.kubernetes.io/name: catalogd olm.operatorframework.io/catalogd-leader: "true" --- +# Source: olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml +apiVersion: v1 +kind: Service +metadata: + annotations: + olm.operatorframework.io/feature-set: experimental + labels: + app.kubernetes.io/name: catalogd + app.kubernetes.io/part-of: olm + name: catalogd-webhook-service + namespace: olmv1-system +spec: + ports: + - name: webhook + port: 9443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/name: catalogd +--- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 kind: Service @@ -2509,6 +2525,8 @@ spec: - --feature-gates=GraphQLCatalogQueries=true - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/certs/tls.crt + - --webhook-tls-key=/var/certs/tls.key - --pull-cas-dir=/var/ca-certs command: - ./catalogd @@ -2804,6 +2822,8 @@ spec: - localhost - catalogd-service.olmv1-system.svc - catalogd-service.olmv1-system.svc.cluster.local + - catalogd-webhook-service.olmv1-system.svc + - catalogd-webhook-service.olmv1-system.svc.cluster.local issuerRef: group: cert-manager.io kind: ClusterIssuer @@ -2883,7 +2903,7 @@ webhooks: - v1 clientConfig: service: - name: catalogd-service + name: catalogd-webhook-service namespace: olmv1-system path: /mutate-olm-operatorframework-io-v1-clustercatalog port: 9443 diff --git a/manifests/standard-e2e.yaml b/manifests/standard-e2e.yaml index 55f46d3262..69df95d929 100644 --- a/manifests/standard-e2e.yaml +++ b/manifests/standard-e2e.yaml @@ -1627,10 +1627,6 @@ spec: port: 443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 - name: metrics port: 7443 protocol: TCP @@ -1639,6 +1635,26 @@ spec: app.kubernetes.io/name: catalogd olm.operatorframework.io/catalogd-leader: "true" --- +# Source: olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml +apiVersion: v1 +kind: Service +metadata: + annotations: + olm.operatorframework.io/feature-set: standard-e2e + labels: + app.kubernetes.io/name: catalogd + app.kubernetes.io/part-of: olm + name: catalogd-webhook-service + namespace: olmv1-system +spec: + ports: + - name: webhook + port: 9443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/name: catalogd +--- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 kind: Service @@ -1742,6 +1758,8 @@ spec: - --feature-gates=APIV1MetasHandler=false - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/certs/tls.crt + - --webhook-tls-key=/var/certs/tls.key - --pull-cas-dir=/var/ca-certs - --tls-profile=custom - --tls-custom-version=TLSv1.3 @@ -2063,6 +2081,8 @@ spec: - localhost - catalogd-service.olmv1-system.svc - catalogd-service.olmv1-system.svc.cluster.local + - catalogd-webhook-service.olmv1-system.svc + - catalogd-webhook-service.olmv1-system.svc.cluster.local issuerRef: group: cert-manager.io kind: ClusterIssuer @@ -2142,7 +2162,7 @@ webhooks: - v1 clientConfig: service: - name: catalogd-service + name: catalogd-webhook-service namespace: olmv1-system path: /mutate-olm-operatorframework-io-v1-clustercatalog port: 9443 diff --git a/manifests/standard.yaml b/manifests/standard.yaml index 29a7c6bc76..fe116cafd6 100644 --- a/manifests/standard.yaml +++ b/manifests/standard.yaml @@ -1588,10 +1588,6 @@ spec: port: 443 protocol: TCP targetPort: 8443 - - name: webhook - port: 9443 - protocol: TCP - targetPort: 9443 - name: metrics port: 7443 protocol: TCP @@ -1600,6 +1596,26 @@ spec: app.kubernetes.io/name: catalogd olm.operatorframework.io/catalogd-leader: "true" --- +# Source: olmv1/templates/service-olmv1-system-catalogd-webhook-service.yml +apiVersion: v1 +kind: Service +metadata: + annotations: + olm.operatorframework.io/feature-set: standard + labels: + app.kubernetes.io/name: catalogd + app.kubernetes.io/part-of: olm + name: catalogd-webhook-service + namespace: olmv1-system +spec: + ports: + - name: webhook + port: 9443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/name: catalogd +--- # Source: olmv1/templates/service-olmv1-system-operator-controller-service.yml apiVersion: v1 kind: Service @@ -1661,6 +1677,8 @@ spec: - --feature-gates=APIV1MetasHandler=false - --tls-cert=/var/certs/tls.crt - --tls-key=/var/certs/tls.key + - --webhook-tls-cert=/var/certs/tls.crt + - --webhook-tls-key=/var/certs/tls.key - --pull-cas-dir=/var/ca-certs command: - ./catalogd @@ -1956,6 +1974,8 @@ spec: - localhost - catalogd-service.olmv1-system.svc - catalogd-service.olmv1-system.svc.cluster.local + - catalogd-webhook-service.olmv1-system.svc + - catalogd-webhook-service.olmv1-system.svc.cluster.local issuerRef: group: cert-manager.io kind: ClusterIssuer @@ -2035,7 +2055,7 @@ webhooks: - v1 clientConfig: service: - name: catalogd-service + name: catalogd-webhook-service namespace: olmv1-system path: /mutate-olm-operatorframework-io-v1-clustercatalog port: 9443 From b43ea8d275c337c3d301e961c80f64238f2d64f2 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Tue, 29 Sep 2026 14:49:01 -0400 Subject: [PATCH 3/3] fix(catalogd): clear stale serving status on unpack failure Mark a catalog unavailable when unpacking fails and no local content remains so a new leader can continue serving other catalogs. Signed-off-by: Todd Short --- .../core/clustercatalog_controller.go | 6 ++ .../core/clustercatalog_controller_test.go | 67 ++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/internal/catalogd/controllers/core/clustercatalog_controller.go b/internal/catalogd/controllers/core/clustercatalog_controller.go index fedfe500f0..ba412046a3 100644 --- a/internal/catalogd/controllers/core/clustercatalog_controller.go +++ b/internal/catalogd/controllers/core/clustercatalog_controller.go @@ -257,6 +257,9 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *ocv1. if err != nil { unpackErr := fmt.Errorf("source catalog content: %w", err) updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), unpackErr) + if !r.Storage.ContentExists(catalog.Name) { + updateStatusNotServing(&catalog.Status, catalog.GetGeneration()) + } return ctrl.Result{}, unpackErr } @@ -266,6 +269,9 @@ func (r *ClusterCatalogReconciler) reconcile(ctx context.Context, catalog *ocv1. if err := r.Storage.Store(ctx, catalog.Name, fsys); err != nil { storageErr := fmt.Errorf("error storing fbc: %v", err) updateStatusProgressing(&catalog.Status, catalog.GetGeneration(), storageErr) + if !r.Storage.ContentExists(catalog.Name) { + updateStatusNotServing(&catalog.Status, catalog.GetGeneration()) + } return ctrl.Result{}, storageErr } baseURL := r.Storage.BaseURL(catalog.Name) diff --git a/internal/catalogd/controllers/core/clustercatalog_controller_test.go b/internal/catalogd/controllers/core/clustercatalog_controller_test.go index f6cbe46dfb..bb625ebe12 100644 --- a/internal/catalogd/controllers/core/clustercatalog_controller_test.go +++ b/internal/catalogd/controllers/core/clustercatalog_controller_test.go @@ -27,6 +27,10 @@ import ( ) func newMockStore(ctrl *gomock.Controller, shouldError bool) *mockstorage.MockInstance { + return newMockStoreWithContent(ctrl, shouldError, true) +} + +func newMockStoreWithContent(ctrl *gomock.Controller, shouldError, contentExists bool) *mockstorage.MockInstance { m := mockstorage.NewMockInstance(ctrl) if shouldError { m.EXPECT().Store(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("mockstore store error")).AnyTimes() @@ -36,10 +40,71 @@ func newMockStore(ctrl *gomock.Controller, shouldError bool) *mockstorage.MockIn m.EXPECT().Delete(gomock.Any()).Return(nil).AnyTimes() } m.EXPECT().BaseURL(gomock.Any()).Return("URL").AnyTimes() - m.EXPECT().ContentExists(gomock.Any()).Return(true).AnyTimes() + m.EXPECT().ContentExists(gomock.Any()).Return(contentExists).AnyTimes() return m } +func TestCatalogdControllerReconcileClearsServingWhenContentIsMissing(t *testing.T) { + ref := mustRef(t, "my.org/someimage@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + for name, tc := range map[string]struct { + puller imageutil.Puller + storeError bool + }{ + "image pull fails": { + puller: &imageutil.FakePuller{Error: errors.New("mock pull error")}, + }, + "storage fails": { + puller: &imageutil.FakePuller{ImageFS: &fstest.MapFS{}, Ref: ref}, + storeError: true, + }, + } { + t.Run(name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + catalog := &ocv1.ClusterCatalog{ + ObjectMeta: metav1.ObjectMeta{ + Name: "catalog", + Generation: 2, + Finalizers: []string{fbcDeletionFinalizer}, + }, + Spec: ocv1.ClusterCatalogSpec{ + Source: ocv1.CatalogSource{ + Type: ocv1.SourceTypeImage, + Image: &ocv1.ImageSource{Ref: "my.org/someimage:latest"}, + }, + }, + Status: ocv1.ClusterCatalogStatus{ + URLs: &ocv1.ClusterCatalogURLs{Base: "URL"}, + LastUnpacked: ptr.To(metav1.Now()), + ResolvedSource: &ocv1.ResolvedCatalogSource{ + Type: ocv1.SourceTypeImage, + Image: &ocv1.ResolvedImageSource{Ref: ref.String()}, + }, + Conditions: []metav1.Condition{ + {Type: ocv1.TypeServing, Status: metav1.ConditionTrue, Reason: ocv1.ReasonAvailable}, + }, + }, + } + reconciler := &ClusterCatalogReconciler{ + ImagePuller: tc.puller, + ImageCache: &imageutil.FakeCache{}, + Storage: newMockStoreWithContent(mockCtrl, tc.storeError, false), + storedCatalogs: map[string]storedCatalogData{}, + } + require.NoError(t, reconciler.setupFinalizers()) + + _, err := reconciler.reconcile(context.Background(), catalog) + require.Error(t, err) + require.Nil(t, catalog.Status.URLs) + require.Nil(t, catalog.Status.LastUnpacked) + require.Nil(t, catalog.Status.ResolvedSource) + serving := meta.FindStatusCondition(catalog.Status.Conditions, ocv1.TypeServing) + require.NotNil(t, serving) + assert.Equal(t, metav1.ConditionFalse, serving.Status) + assert.Equal(t, ocv1.ReasonUnavailable, serving.Reason) + }) + } +} + func TestCatalogdControllerReconcile(t *testing.T) { mockCtrl := gomock.NewController(t) for _, tt := range []struct {