diff --git a/pkg/crd-installer/installer.go b/pkg/crd-installer/installer.go index ffc16d01..03021264 100644 --- a/pkg/crd-installer/installer.go +++ b/pkg/crd-installer/installer.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "os" "slices" "sync" @@ -22,6 +23,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/util/retry" + "github.com/deckhouse/module-sdk/pkg/crd-installer/openapi" "github.com/deckhouse/module-sdk/pkg/utils" ) @@ -164,6 +166,8 @@ func (cp *CRDsInstaller) processCRD(ctx context.Context, crdFilePath string) err crdReader := apimachineryYaml.NewDocumentDecoder(crdFileReader) + var errs error + for { n, err := crdReader.Read(cp.buffer) if err != nil { @@ -171,7 +175,9 @@ func (cp *CRDsInstaller) processCRD(ctx context.Context, crdFilePath string) err break } - return err + // the documents read so far may already have failed: reporting only the read + // error would hide them + return errors.Join(errs, err) } data := cp.buffer[:n] @@ -182,18 +188,19 @@ func (cp *CRDsInstaller) processCRD(ctx context.Context, crdFilePath string) err rd := bytes.NewReader(data) - err = cp.putCRDToCluster(ctx, rd, n) - if err != nil { - return err + // one bad document must not skip the rest of the file: the documents after it are + // unrelated CRDs, and skipping them silently leaves the module without them + if err := cp.putCRDToCluster(ctx, rd, n); err != nil { + errs = errors.Join(errs, err) } } - return nil + return errs } func (cp *CRDsInstaller) putCRDToCluster(ctx context.Context, crdReader io.Reader, bufferSize int) error { - // Decode into unstructured so vendor schema extensions (x-kubernetes-*, x-ui-*, ...) - // survive verbatim; the typed struct below is used only to read metadata. + // Decode into unstructured first: the typed struct below cannot hold + // x-kubernetes-sensitive-data, and sanitize needs the original document to recover it. desired := &unstructured.Unstructured{} err := apimachineryYaml.NewYAMLOrJSONDecoder(crdReader, bufferSize).Decode(&desired) if err != nil { @@ -213,6 +220,19 @@ func (cp *CRDsInstaller) putCRDToCluster(ctx context.Context, crdReader io.Reade return fmt.Errorf("invalid CRD document apiversion/kind: '%s/%s'", crd.APIVersion, crd.Kind) } + if err := applyServerDefaults(crd, desired); err != nil { + return fmt.Errorf("default %s: %w", crd.Name, err) + } + + // a schema this build cannot decode must not keep the CRD out of the cluster: the + // document is queued as it came — what the installer did before it pruned anything — + // and the error is reported afterwards. The apiserver prunes the key it does not + // understand, while a missing CRD takes every custom resource of that kind with it. + sanitizeErr := sanitize(desired) + if sanitizeErr != nil { + sanitizeErr = fmt.Errorf("sanitize %s: %w", crd.Name, sanitizeErr) + } + cp.k8sTasks.Go(func() error { err := cp.updateOrInsertCRD(ctx, crd, desired) if err == nil { @@ -251,15 +271,120 @@ func (cp *CRDsInstaller) putCRDToCluster(ctx context.Context, crdReader io.Reade return err }) + return sanitizeErr +} + +// sanitize drops the schema keys the apiserver does not know from the CRD document. +// +// x-doc-examples and friends, plus keys that look official but are not, like +// x-kubernetes-immutable, are removed here instead of being sent. The apiserver prunes +// them anyway, after logging one "unknown field" warning per occurrence, and because it +// prunes them the stored spec could never equal the desired one, so every run issued a +// pointless Update. +// +// Only the version schemas are rewritten; the rest of the document is passed through +// untouched. Rebuilding it from apiextensionsv1.CustomResourceDefinition instead would +// pin the installer to the CRD fields of the apiextensions-apiserver it was compiled +// against, and silently strip anything a newer or patched apiserver understands. +func sanitize(desired *unstructured.Unstructured) error { + versions, ok := nestedValue(desired.Object, "spec", "versions").([]any) + if !ok { + // no versions, or not a list: let the apiserver reject the document + return nil + } + + for _, version := range versions { + versionMap, ok := version.(map[string]any) + if !ok { + continue + } + + schema, ok := versionMap["schema"].(map[string]any) + if !ok { + continue + } + + rawSchema, ok := schema["openAPIV3Schema"].(map[string]any) + if !ok { + continue + } + + cleanSchema, err := openapi.Prune(rawSchema) + if err != nil { + name, _, _ := unstructured.NestedString(versionMap, "name") + + return fmt.Errorf("version %q schema: %w", name, err) + } + + schema["openAPIV3Schema"] = cleanSchema + } + + return nil +} + +// nestedValue returns the value at the given path without copying it, or nil if the path +// does not lead to one. Errors are the same as absence for every caller here. +func nestedValue(obj map[string]any, fields ...string) any { + value, found, err := unstructured.NestedFieldNoCopy(obj, fields...) + if err != nil || !found { + return nil + } + + return value +} + +// applyServerDefaults writes the .spec fields the apiserver fills in itself into the +// document, so a manifest that omits them does not differ from the stored object on +// every single reconcile. +// +// ponytail: only the .spec fields known to churn are written here; if a CRD still updates +// on every reconcile, diff the stored object against the manifest and add the field that +// differs. +func applyServerDefaults(crd *apiextensionsv1.CustomResourceDefinition, desired *unstructured.Unstructured) error { + apiextensionsv1.SetDefaults_CustomResourceDefinitionSpec(&crd.Spec) + + // served and storage have no omitempty upstream, so the stored object always carries + // both on every version — a manifest that omits either would differ from it forever + versions, _ := nestedValue(desired.Object, "spec", "versions").([]any) + for _, version := range versions { + versionMap, ok := version.(map[string]any) + if !ok { + continue + } + + for _, field := range []string{"served", "storage"} { + if _, ok := versionMap[field]; !ok { + versionMap[field] = false + } + } + } + + // spec.conversion is defaulted too, but updateOrInsertCRD always takes the in-cluster + // one, which the apiserver has already defaulted + names := map[string]string{ + "singular": crd.Spec.Names.Singular, + "listKind": crd.Spec.Names.ListKind, + } + + for field, value := range names { + if value == "" { + continue + } + + if err := unstructured.SetNestedField(desired.Object, value, "spec", "names", field); err != nil { + return fmt.Errorf("set spec.names.%s: %w", field, err) + } + } + return nil } func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *apiextensionsv1.CustomResourceDefinition, desired *unstructured.Unstructured) error { return retry.RetryOnConflict(retry.DefaultRetry, func() error { + desired.SetLabels(overlay(desired.GetLabels(), cp.crdExtraLabels)) + existing, err := cp.k8sClient.Resource(crdGVR).Get(ctx, crd.GetName(), apimachineryv1.GetOptions{}) if apierrors.IsNotFound(err) { - mergeLabels(desired, cp.crdExtraLabels) - _, err = cp.k8sClient.Resource(crdGVR).Create(ctx, desired, apimachineryv1.CreateOptions{}) if err != nil { return fmt.Errorf("create crd: %w", err) @@ -317,8 +442,6 @@ func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *apiextensio } } - mergeLabels(desired, cp.crdExtraLabels) - desiredSpec, _, err := unstructured.NestedMap(desired.Object, "spec") if err != nil { return fmt.Errorf("read desired spec: %w", err) @@ -329,12 +452,27 @@ func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *apiextensio return fmt.Errorf("read existing spec: %w", err) } - // diff on lossless unstructured specs so vendor extensions are not silently dropped - // ponytail: apiserver-defaulted .spec fields may differ from the manifest and cause - // reconcile churn; the update stays idempotent, tighten the diff here if it ever churns. + // labels and annotations belong to whoever wrote them: dropping + // app.kubernetes.io/managed-by or meta.helm.sh/release-name breaks the next helm + // upgrade of the chart that installed the CRD, so both maps are overlaid, never + // replaced. The result is what the object must end up holding, which is also what + // makes the comparison below exact. + // + // ponytail: overlaying means a key the manifest stops declaring stays in the + // cluster — retracting one needs the field ownership the apiserver keeps for + // server-side apply; switch this whole update to Apply if that ever matters. + labels := overlay(existing.GetLabels(), desired.GetLabels()) + annotations := overlay(existing.GetAnnotations(), desired.GetAnnotations()) + + // The specs are compared, not merged: the desired one is pruned by sanitize and the + // existing one by the apiserver, so a manifest applied over the state the apiserver + // derived from it is a no-op. A .spec key this cluster's apiserver does not know is + // the exception — it prunes the key while the desired document keeps it, so that CRD + // is updated on every reconcile. Deliberate: the key is sent so an apiserver that + // does know it gets it. if cmp.Equal(existingSpec, desiredSpec) && - cmp.Equal(existing.GetLabels(), desired.GetLabels()) && - cmp.Equal(existing.GetAnnotations(), desired.GetAnnotations()) { + cmp.Equal(existing.GetLabels(), labels) && + cmp.Equal(existing.GetAnnotations(), annotations) { return nil } @@ -343,8 +481,8 @@ func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *apiextensio if err := unstructured.SetNestedMap(existing.Object, desiredSpec, "spec"); err != nil { return fmt.Errorf("set spec: %w", err) } - existing.SetLabels(desired.GetLabels()) - existing.SetAnnotations(desired.GetAnnotations()) + existing.SetLabels(labels) + existing.SetAnnotations(annotations) existing.SetResourceVersion(resourceVersion) _, err = cp.k8sClient.Resource(crdGVR).Update(ctx, existing, apimachineryv1.UpdateOptions{}) @@ -356,17 +494,22 @@ func (cp *CRDsInstaller) updateOrInsertCRD(ctx context.Context, crd *apiextensio }) } -func mergeLabels(u *unstructured.Unstructured, extra map[string]string) { - labels := u.GetLabels() - if labels == nil { - labels = make(map[string]string, len(extra)) +// overlay returns base with over's keys written on top of it. A key base holds and over +// does not is kept. +// +// nil in, nil out: the apiserver returns no map at all for empty labels or annotations, and +// an empty one written back would never compare equal to that — SetLabels/SetAnnotations +// remove the field for nil and set metadata.labels: {} for an empty map. +func overlay(base, over map[string]string) map[string]string { + if len(over) == 0 { + return base } - for k, v := range extra { - labels[k] = v - } + out := make(map[string]string, len(base)+len(over)) + maps.Copy(out, base) + maps.Copy(out, over) - u.SetLabels(labels) + return out } func (cp *CRDsInstaller) GetCRDFromCluster(ctx context.Context, crdName string) (*apiextensionsv1.CustomResourceDefinition, error) { diff --git a/pkg/crd-installer/installer_test.go b/pkg/crd-installer/installer_test.go index 227fc537..9f5401bf 100644 --- a/pkg/crd-installer/installer_test.go +++ b/pkg/crd-installer/installer_test.go @@ -12,9 +12,78 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/json" "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" ) +func countUpdates(actions []k8stesting.Action, name string) int { + var n int + + for _, action := range actions { + // k8stesting.CreateAction and UpdateAction are the same interface, so a create + // satisfies both: the verb is the only way to tell them apart. The subresource check + // keeps the storedVersions status write out of the count. + if action.GetVerb() != "update" || action.GetSubresource() != "" { + continue + } + + obj, ok := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + if ok && obj.GetName() == name { + n++ + } + } + + return n +} + +// storeAsWire makes the fake client behave the way the wire does. The tracker keeps the +// exact Go values it is handed, so without this an object built with float64 numbers +// reads back as float64 and a test can pass on state a real apiserver would never +// return. +func storeAsWire(fc *fake.FakeDynamicClient) { + react := func(action k8stesting.Action) (bool, runtime.Object, error) { + withObject, ok := action.(interface{ GetObject() runtime.Object }) + if !ok { + return false, nil, nil + } + + obj, ok := withObject.GetObject().(*unstructured.Unstructured) + if !ok { + return false, nil, nil + } + + // the reactor runs on the installer's worker goroutines, so an error is returned to + // the caller rather than asserted: require.NoError would call t.FailNow off the test + // goroutine, which unwinds that worker and reports nothing about the CRD it dropped + data, err := json.Marshal(obj.Object) + if err != nil { + return true, nil, err + } + + wire := map[string]any{} + if err := json.Unmarshal(data, &wire); err != nil { + return true, nil, err + } + + // ObjectMeta.Labels/Annotations are omitempty, so the apiserver never returns an + // empty map for them — it returns nothing + for _, field := range []string{"labels", "annotations"} { + if m, ok := nestedValue(wire, "metadata", field).(map[string]any); ok && len(m) == 0 { + unstructured.RemoveNestedField(wire, "metadata", field) + } + } + + obj.Object = wire + + // fall through to the object tracker, which stores a deep copy of what we just fixed + return false, nil, nil + } + + fc.PrependReactor("create", "*", react) + fc.PrependReactor("update", "*", react) +} + func TestCRDInstaller(t *testing.T) { crdScheme := runtime.NewScheme() @@ -30,6 +99,7 @@ func TestCRDInstaller(t *testing.T) { } fc := fake.NewSimpleDynamicClient(crdScheme) + storeAsWire(fc) t.Run("install CRD", func(t *testing.T) { inst := NewCRDsInstaller(fc, []string{"testdata/1_example.yaml"}, WithExtraLabels(map[string]string{"heritage": "deckhouse"})) @@ -58,7 +128,9 @@ func TestCRDInstaller(t *testing.T) { un, err := fc.Resource(gvr).Get(context.Background(), "widgets.example.com", apimachineryv1.GetOptions{}) require.NoError(t, err) - assert.Equal(t, map[string]string{"foo": "bar", "one": "new", "another": "lab"}, un.GetLabels()) + // heritage is from the previous run: labels are overlaid, so a key this run no longer + // declares is left in place rather than deleted + assert.Equal(t, map[string]string{"foo": "bar", "one": "new", "another": "lab", "heritage": "deckhouse"}, un.GetLabels()) assert.Equal(t, map[string]string{"bar": "baz", "two": "new"}, un.GetAnnotations()) var crd v1.CustomResourceDefinition err = runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &crd) @@ -148,6 +220,166 @@ func TestCRDInstaller(t *testing.T) { assert.Equal(t, true, token["x-kubernetes-sensitive-data"]) }) + // Regression: keys the apiserver does not know must never be sent. It prunes them + // anyway, one "unknown field" warning per occurrence, and the pruning is what made + // the desired spec permanently differ from the stored one. + // + // This only checks that sanitize is reached from Run and applies to the whole schema + // tree; which keys survive at which nesting position is openapi.Prune's own job and is + // covered by TestRoundTrip* in that package. + t.Run("strips unknown schema extensions", func(t *testing.T) { + inst := NewCRDsInstaller(fc, []string{"testdata/6_unknown_extensions.yaml"}) + require.NoError(t, inst.Run(context.Background())) + + un, err := fc.Resource(gvr).Get(context.Background(), "extensions.example.com", apimachineryv1.GetOptions{}) + require.NoError(t, err) + + versions, _, err := unstructured.NestedSlice(un.Object, "spec", "versions") + require.NoError(t, err) + + root, found, err := unstructured.NestedMap(versions[0].(map[string]any), "schema", "openAPIV3Schema") + require.NoError(t, err) + require.True(t, found) + + assert.NotContains(t, root, "x-doc-examples") + + token, found, err := unstructured.NestedMap(root, "properties", "spec", "properties", "token") + require.NoError(t, err) + require.True(t, found) + + assert.Equal(t, true, token["x-kubernetes-sensitive-data"], "the one extension that must survive") + assert.NotContains(t, token, "x-doc-examples") + }) + + // Regression: a CRD field this build's apiextensions-apiserver does not model must + // still reach an apiserver that understands it — sanitize prunes schemas, not the + // document. + t.Run("keeps CRD fields outside the schema", func(t *testing.T) { + inst := NewCRDsInstaller(fc, []string{"testdata/9_unknown_crd_field.yaml"}) + require.NoError(t, inst.Run(context.Background())) + + un, err := fc.Resource(gvr).Get(context.Background(), "futures.example.com", apimachineryv1.GetOptions{}) + require.NoError(t, err) + + versions, _, err := unstructured.NestedSlice(un.Object, "spec", "versions") + require.NoError(t, err) + + assert.Equal(t, "a field from a newer apiserver", versions[0].(map[string]any)["fieldFromTheFuture"]) + assert.NotContains(t, un.Object, "status", "the manifest declares no status, so none must be sent") + + // the manifest has no labels and the installer adds none, so the second run must see + // the nil the cluster returns as equal to what it would write + before := countUpdates(fc.Actions(), "futures.example.com") + + require.NoError(t, NewCRDsInstaller(fc, []string{"testdata/9_unknown_crd_field.yaml"}).Run(context.Background())) + + assert.Equal(t, before, countUpdates(fc.Actions(), "futures.example.com"), + "a CRD without labels must not churn") + }) + + // Regression: applying a manifest over the state the apiserver derives from it must be + // a no-op. The apiserver prunes the x-doc-* keys, defaults spec.names and returns whole + // numbers as int64, so before the fix the desired spec could never equal the stored one + // and every single run issued a full Update of every CRD. + // + // The fake client prunes and defaults nothing, so the derived state has to be installed + // explicitly — reinstalling the same file twice would pass either way and prove nothing. + t.Run("applying a manifest over its stored form does not update", func(t *testing.T) { + inst := NewCRDsInstaller(fc, []string{"testdata/7_churn_stored.yaml"}) + require.NoError(t, inst.Run(context.Background())) + + require.Zero(t, countUpdates(fc.Actions(), "churns.example.com"), + "a CRD that is not in the cluster is created, not updated") + + inst = NewCRDsInstaller(fc, []string{"testdata/8_churn_manifest.yaml"}) + require.NoError(t, inst.Run(context.Background())) + + assert.Zero(t, countUpdates(fc.Actions(), "churns.example.com"), + "the doc keys, the defaulted names and the numeric bounds must not read as a diff") + }) + + // Regression: sanitize runs before the CRD is queued, so an error from it used to abort + // the whole file and silently skip every document after the bad one. It must not keep the + // CRD it failed on out of the cluster either: the apiserver prunes the key it cannot + // read, while a missing CRD takes every custom resource of that kind with it. + t.Run("a bad document does not skip the rest of the file", func(t *testing.T) { + inst := NewCRDsInstaller(fc, []string{"testdata/10_multi_document.yaml"}) + require.Error(t, inst.Run(context.Background()), "the broken document must still be reported") + + for _, name := range []string{"firsts.example.com", "broken.example.com", "lasts.example.com"} { + _, err := fc.Resource(gvr).Get(context.Background(), name, apimachineryv1.GetOptions{}) + require.NoError(t, err, "%s is in the same file and must be installed", name) + } + + un, err := fc.Resource(gvr).Get(context.Background(), "broken.example.com", apimachineryv1.GetOptions{}) + require.NoError(t, err) + + versions, _, err := unstructured.NestedSlice(un.Object, "spec", "versions") + require.NoError(t, err) + + token, found, err := unstructured.NestedMap(versions[0].(map[string]any), + "schema", "openAPIV3Schema", "properties", "token") + require.NoError(t, err) + require.True(t, found) + + assert.Equal(t, "yes", token["x-kubernetes-sensitive-data"], + "a schema the fork cannot decode is sent as it came, for the apiserver to prune") + }) + + // Regression: labels and annotations belong to whoever wrote them. Replacing either map + // wholesale dropped the Helm ownership keys — app.kubernetes.io/managed-by among the + // labels, meta.helm.sh/* among the annotations — and the next helm upgrade of that chart + // then failed on "invalid ownership metadata". + t.Run("keeps labels and annotations written by other actors", func(t *testing.T) { + // own client: the CRD must be reached by the update path with nothing but the seeded + // state, otherwise a leftover from another subtest can be what forces the update + fc := fake.NewSimpleDynamicClient(crdScheme) + storeAsWire(fc) + + // widgets.example.com as a helm chart installed it + seed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]any{ + "name": "widgets.example.com", + "labels": map[string]any{"app.kubernetes.io/managed-by": "Helm"}, + "annotations": map[string]any{"meta.helm.sh/release-name": "some-chart"}, + }, + "spec": map[string]any{ + "group": "example.com", + "names": map[string]any{ + "kind": "Widget", "listKind": "WidgetList", "plural": "widgets", "singular": "widget", + }, + "scope": "Namespaced", + "versions": []any{map[string]any{"name": "v1", "served": true, "storage": true}}, + }, + }} + _, err := fc.Resource(gvr).Create(context.Background(), seed, apimachineryv1.CreateOptions{}) + require.NoError(t, err) + + extra := WithExtraLabels(map[string]string{"heritage": "deckhouse"}) + require.NoError(t, NewCRDsInstaller(fc, []string{"testdata/2_example.yaml"}, extra).Run(context.Background())) + + un, err := fc.Resource(gvr).Get(context.Background(), "widgets.example.com", apimachineryv1.GetOptions{}) + require.NoError(t, err) + + assert.Equal(t, map[string]string{ + "foo": "bar", "one": "new", "heritage": "deckhouse", + "app.kubernetes.io/managed-by": "Helm", + }, un.GetLabels(), "the helm ownership label must survive the update") + assert.Equal(t, map[string]string{ + "bar": "baz", "two": "new", + "meta.helm.sh/release-name": "some-chart", + }, un.GetAnnotations(), "the helm ownership annotation must survive the update") + + before := countUpdates(fc.Actions(), "widgets.example.com") + + require.NoError(t, NewCRDsInstaller(fc, []string{"testdata/2_example.yaml"}, extra).Run(context.Background())) + + assert.Equal(t, before, countUpdates(fc.Actions(), "widgets.example.com"), + "foreign labels and annotations must not read as a diff either") + }) + // Regression: a comment-only yaml document decodes to a nil object and must be skipped, // not panic on the nil dereference. t.Run("skips comment-only documents", func(t *testing.T) { diff --git a/pkg/crd-installer/openapi/marshal.go b/pkg/crd-installer/openapi/marshal.go new file mode 100644 index 00000000..d3b24313 --- /dev/null +++ b/pkg/crd-installer/openapi/marshal.go @@ -0,0 +1,130 @@ +package openapi + +import ( + "errors" + + "k8s.io/apimachinery/pkg/util/json" +) + +// Ported from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/marshal.go. +// The CBOR halves are left out: these types are only ever an intermediate step +// between two map[string]interface{} values and never cross the wire themselves. + +var ( + jsTrue = []byte("true") + jsFalse = []byte("false") +) + +func (s JSONSchemaPropsOrBool) MarshalJSON() ([]byte, error) { + if s.Schema != nil { + return json.Marshal(s.Schema) + } + + if s.Schema == nil && !s.Allows { + return jsFalse, nil + } + + return jsTrue, nil +} + +func (s *JSONSchemaPropsOrBool) UnmarshalJSON(data []byte) error { + var nw JSONSchemaPropsOrBool + + switch { + case len(data) == 0: + case data[0] == '{': + var sch JSONSchemaProps + if err := json.Unmarshal(data, &sch); err != nil { + return err + } + + nw.Allows = true + nw.Schema = &sch + case len(data) == 4 && string(data) == "true": + nw.Allows = true + case len(data) == 5 && string(data) == "false": + nw.Allows = false + default: + return errors.New("boolean or JSON schema expected") + } + + *s = nw + + return nil +} + +func (s JSONSchemaPropsOrStringArray) MarshalJSON() ([]byte, error) { + if len(s.Property) > 0 { + return json.Marshal(s.Property) + } + + if s.Schema != nil { + return json.Marshal(s.Schema) + } + + return []byte("null"), nil +} + +func (s *JSONSchemaPropsOrStringArray) UnmarshalJSON(data []byte) error { + var first byte + if len(data) > 1 { + first = data[0] + } + + var nw JSONSchemaPropsOrStringArray + + if first == '{' { + var sch JSONSchemaProps + if err := json.Unmarshal(data, &sch); err != nil { + return err + } + + nw.Schema = &sch + } + + if first == '[' { + if err := json.Unmarshal(data, &nw.Property); err != nil { + return err + } + } + + *s = nw + + return nil +} + +func (s JSONSchemaPropsOrArray) MarshalJSON() ([]byte, error) { + if len(s.JSONSchemas) > 0 { + return json.Marshal(s.JSONSchemas) + } + + return json.Marshal(s.Schema) +} + +func (s *JSONSchemaPropsOrArray) UnmarshalJSON(data []byte) error { + var nw JSONSchemaPropsOrArray + + var first byte + if len(data) > 1 { + first = data[0] + } + + if first == '{' { + var sch JSONSchemaProps + if err := json.Unmarshal(data, &sch); err != nil { + return err + } + + nw.Schema = &sch + } + + if first == '[' { + if err := json.Unmarshal(data, &nw.JSONSchemas); err != nil { + return err + } + } + + *s = nw + + return nil +} diff --git a/pkg/crd-installer/openapi/marshal_test.go b/pkg/crd-installer/openapi/marshal_test.go new file mode 100644 index 00000000..e1226a1a --- /dev/null +++ b/pkg/crd-installer/openapi/marshal_test.go @@ -0,0 +1,58 @@ +package openapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" +) + +// TestForkMarshalsLikeUpstream is the guard the reflection tests in types_test.go cannot +// be. Every field of the union types is json:"-", so the bytes they produce are decided by +// the hand-ported marshallers in marshal.go alone: a drift there — null instead of [], a +// lost Allows — changes what the installer applies to every CRD using that keyword while +// both field-set comparisons still pass. Only the bytes show it. +func TestForkMarshalsLikeUpstream(t *testing.T) { + // x-kubernetes-sensitive-data is left out on purpose: the upstream type cannot hold it, + // and TestRoundTripKeepsKnownFields already covers it + raw := map[string]any{ + "type": "object", + "properties": map[string]any{ + // items, single-schema form + "tags": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + // items, tuple form + "pair": map[string]any{"type": "array", "items": []any{ + map[string]any{"type": "string"}, + map[string]any{"type": "integer"}, + }}, + // additionalProperties in all three forms + "labels": map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"}}, + "free": map[string]any{"type": "object", "additionalProperties": true}, + "closed": map[string]any{"type": "object", "additionalProperties": false}, + "list": map[string]any{"type": "array", "additionalItems": true}, + // dependencies in both forms + "deps": map[string]any{"type": "object", "dependencies": map[string]any{ + "needsOther": []any{"other"}, + "needsShape": map[string]any{"type": "string"}, + }}, + }, + } + + fork := &JSONSchemaProps{} + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(raw, fork)) + + upstream := &apiextensionsv1.JSONSchemaProps{} + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(raw, upstream)) + + forkJSON, err := json.Marshal(fork) + require.NoError(t, err) + + upstreamJSON, err := json.Marshal(upstream) + require.NoError(t, err) + + assert.JSONEq(t, string(upstreamJSON), string(forkJSON), + "port the upstream marshal.go change into this package's") +} diff --git a/pkg/crd-installer/openapi/prune.go b/pkg/crd-installer/openapi/prune.go new file mode 100644 index 00000000..2648c862 --- /dev/null +++ b/pkg/crd-installer/openapi/prune.go @@ -0,0 +1,35 @@ +package openapi + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" +) + +// Prune returns the schema with every key that is not a field of JSONSchemaProps +// removed, at every nesting level. +// +// The result is encoded through JSON rather than through the reflection converter so +// that whole numbers come back as int64, exactly as they arrive from the apiserver. +// Reflection encodes maximum/minimum/multipleOf as float64 — the Go type of those +// fields — and a desired spec carrying float64 could never compare equal to the stored +// one, so every reconcile would issue an Update. +func Prune(raw map[string]any) (map[string]any, error) { + props := &JSONSchemaProps{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(raw, props); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + + data, err := json.Marshal(props) + if err != nil { + return nil, fmt.Errorf("encode: %w", err) + } + + clean := map[string]any{} + if err := json.Unmarshal(data, &clean); err != nil { + return nil, fmt.Errorf("decode pruned: %w", err) + } + + return clean, nil +} diff --git a/pkg/crd-installer/openapi/types.go b/pkg/crd-installer/openapi/types.go new file mode 100644 index 00000000..1cd091ab --- /dev/null +++ b/pkg/crd-installer/openapi/types.go @@ -0,0 +1,125 @@ +// Package openapi holds the CRD validation schema type that the installer applies +// to the cluster. +// +// It is a fork of apiextensionsv1.JSONSchemaProps. The fork exists for exactly one +// reason: the Deckhouse kube-apiserver carries 010-x-kubernetes-sensitive-data.patch +// and runs with CRDSensitiveData=true, so it understands one schema field that the +// stock apiextensions-apiserver Go types do not. Decoding a CRD through the upstream +// type would drop that field before it ever reached the cluster. +// +// The fork is also the strict contract in the other direction: any key that is not a +// field here — x-doc-examples, x-examples, x-description, x-kubernetes-immutable and +// friends — is dropped on decode instead of being sent to the apiserver, which would +// prune it anyway and log an "unknown field" warning for every occurrence. +// +// The contract holds only on the Deckhouse kube-apiserver. On a stock one — a managed +// control plane, a dev cluster, CRDSensitiveData off — x-kubernetes-sensitive-data is +// pruned server-side, so a CRD that carries it is updated on every reconcile. That is +// accepted: the field is meaningless there anyway, and detecting it would mean probing +// the apiserver build for every install. +// +// Being the allowlist cuts the other way too: a schema field the cluster's apiserver +// understands and this build does not is dropped silently, and TestForkCoversUpstreamFields +// only fires when this module bumps k8s.io/apiextensions-apiserver — never when the cluster +// moves ahead of it. Keep the dependency in step with the apiserver Deckhouse ships. A +// schema this fork cannot decode at all is not dropped: the installer sends that document +// as it came and reports the error (see sanitize). +// +// TestForkCoversUpstreamFields and TestForkMarshalsLikeUpstream guard the copy against +// drift; read them before bumping k8s.io/apiextensions-apiserver. +package openapi + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +// JSONSchemaProps is a JSON-Schema following Specification Draft 4 (http://json-schema.org/). +// +// Field-for-field mirror of apiextensionsv1.JSONSchemaProps, with the nested schema +// positions retargeted at this package and XSensitiveData added. +type JSONSchemaProps struct { + ID string `json:"id,omitempty"` + Schema apiextensionsv1.JSONSchemaURL `json:"$schema,omitempty"` + Ref *string `json:"$ref,omitempty"` + Description string `json:"description,omitempty"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + Title string `json:"title,omitempty"` + Default *apiextensionsv1.JSON `json:"default,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + ExclusiveMaximum bool `json:"exclusiveMaximum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + ExclusiveMinimum bool `json:"exclusiveMinimum,omitempty"` + MaxLength *int64 `json:"maxLength,omitempty"` + MinLength *int64 `json:"minLength,omitempty"` + Pattern string `json:"pattern,omitempty"` + MaxItems *int64 `json:"maxItems,omitempty"` + MinItems *int64 `json:"minItems,omitempty"` + UniqueItems bool `json:"uniqueItems,omitempty"` + MultipleOf *float64 `json:"multipleOf,omitempty"` + + Enum []apiextensionsv1.JSON `json:"enum,omitempty"` + MaxProperties *int64 `json:"maxProperties,omitempty"` + MinProperties *int64 `json:"minProperties,omitempty"` + + Required []string `json:"required,omitempty"` + Items *JSONSchemaPropsOrArray `json:"items,omitempty"` + + AllOf []JSONSchemaProps `json:"allOf,omitempty"` + OneOf []JSONSchemaProps `json:"oneOf,omitempty"` + AnyOf []JSONSchemaProps `json:"anyOf,omitempty"` + Not *JSONSchemaProps `json:"not,omitempty"` + Properties map[string]JSONSchemaProps `json:"properties,omitempty"` + AdditionalProperties *JSONSchemaPropsOrBool `json:"additionalProperties,omitempty"` + PatternProperties map[string]JSONSchemaProps `json:"patternProperties,omitempty"` + Dependencies JSONSchemaDependencies `json:"dependencies,omitempty"` + AdditionalItems *JSONSchemaPropsOrBool `json:"additionalItems,omitempty"` + Definitions JSONSchemaDefinitions `json:"definitions,omitempty"` + ExternalDocs *apiextensionsv1.ExternalDocumentation `json:"externalDocs,omitempty"` + Example *apiextensionsv1.JSON `json:"example,omitempty"` + Nullable bool `json:"nullable,omitempty"` + + XPreserveUnknownFields *bool `json:"x-kubernetes-preserve-unknown-fields,omitempty"` + XEmbeddedResource bool `json:"x-kubernetes-embedded-resource,omitempty"` + XIntOrString bool `json:"x-kubernetes-int-or-string,omitempty"` + XListMapKeys []string `json:"x-kubernetes-list-map-keys,omitempty"` + XListType *string `json:"x-kubernetes-list-type,omitempty"` + XMapType *string `json:"x-kubernetes-map-type,omitempty"` + XValidations apiextensionsv1.ValidationRules `json:"x-kubernetes-validations,omitempty"` + + // XSensitiveData marks a field (or an object/array subtree) as sensitive: the + // apiserver encrypts it in etcd, filters it by RBAC through the /sensitive + // subresource, and masks it in audit logs. + // + // This is NOT an upstream Kubernetes field. It only exists on the Deckhouse + // kube-apiserver, and it is the sole reason this package forks JSONSchemaProps. + // Keep it listed in forkOnlyFields in the test when adding others. + XSensitiveData bool `json:"x-kubernetes-sensitive-data,omitempty"` +} + +// JSONSchemaPropsOrArray represents a value that can either be a JSONSchemaProps +// or an array of JSONSchemaProps. Mainly here for serialization purposes. +type JSONSchemaPropsOrArray struct { + Schema *JSONSchemaProps `json:"-"` + JSONSchemas []JSONSchemaProps `json:"-"` +} + +// JSONSchemaPropsOrBool represents JSONSchemaProps or a boolean value. +// Defaults to true for the boolean property. +type JSONSchemaPropsOrBool struct { + Allows bool `json:"-"` + Schema *JSONSchemaProps `json:"-"` +} + +// JSONSchemaPropsOrStringArray represents a JSONSchemaProps or a string array. +type JSONSchemaPropsOrStringArray struct { + Schema *JSONSchemaProps `json:"-"` + Property []string `json:"-"` +} + +// JSONSchemaDependencies represent a dependencies property. +type JSONSchemaDependencies map[string]JSONSchemaPropsOrStringArray + +// JSONSchemaDefinitions contains the models explicitly defined in this spec. +type JSONSchemaDefinitions map[string]JSONSchemaProps diff --git a/pkg/crd-installer/openapi/types_test.go b/pkg/crd-installer/openapi/types_test.go new file mode 100644 index 00000000..0f9732d0 --- /dev/null +++ b/pkg/crd-installer/openapi/types_test.go @@ -0,0 +1,316 @@ +package openapi + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +// forkOnlyFields are the json keys this package adds on top of upstream +// JSONSchemaProps. Everything else must exist on both sides. +var forkOnlyFields = map[string]struct{}{ + "x-kubernetes-sensitive-data": {}, +} + +// TestForkCoversUpstreamFields is the guard that keeps the fork honest. JSONSchemaProps +// is copied by hand, so a k8s bump that adds a schema field — or retypes one — would +// otherwise make the installer silently strip or mis-decode that field on every CRD it +// applies, with no warning anywhere. If this fails after bumping +// k8s.io/apiextensions-apiserver, port the change into types.go rather than relaxing +// the test. +func TestForkCoversUpstreamFields(t *testing.T) { + upstream := jsonFields(reflect.TypeOf(apiextensionsv1.JSONSchemaProps{})) + fork := jsonFields(reflect.TypeOf(JSONSchemaProps{})) + + for tag, up := range upstream { + f, ok := fork[tag] + if !ok { + t.Errorf("apiextensionsv1.JSONSchemaProps.%s (%q) is missing from the fork: the installer would strip it from every CRD", up.name, tag) + + continue + } + + if want := forkType(up.typ); f.typ != want { + t.Errorf("JSONSchemaProps.%s (%q) is %s upstream, so the fork must declare it %s, not %s: the installer would mis-decode it", up.name, tag, up.typ, want, f.typ) + } + + if f.opts != up.opts { + t.Errorf("JSONSchemaProps.%s (%q) has the json tag options %q upstream but %q in the fork: the installer would serialize it where the apiserver omits it, and every CRD would be updated on every reconcile", up.name, tag, up.opts, f.opts) + } + } + + for tag, f := range fork { + if _, ok := upstream[tag]; ok { + continue + } + + if _, ok := forkOnlyFields[tag]; !ok { + t.Errorf("JSONSchemaProps.%s (%q) exists in neither upstream nor forkOnlyFields: the apiserver will reject it as an unknown field", f.name, tag) + } + } +} + +// TestForkCoversUpstreamUnions guards the three union types. Every one of their fields is +// json:"-" and carried by the hand-ported marshallers in marshal.go, so the tag-based +// guard above never sees them — they are the part of the fork most likely to drift +// unnoticed. +func TestForkCoversUpstreamUnions(t *testing.T) { + for _, tc := range []struct{ fork, upstream reflect.Type }{ + {reflect.TypeOf(JSONSchemaPropsOrArray{}), reflect.TypeOf(apiextensionsv1.JSONSchemaPropsOrArray{})}, + {reflect.TypeOf(JSONSchemaPropsOrBool{}), reflect.TypeOf(apiextensionsv1.JSONSchemaPropsOrBool{})}, + {reflect.TypeOf(JSONSchemaPropsOrStringArray{}), reflect.TypeOf(apiextensionsv1.JSONSchemaPropsOrStringArray{})}, + } { + t.Run(tc.fork.Name(), func(t *testing.T) { + assert.Equal(t, forkStructFields(tc.upstream), structFields(tc.fork), + "port the upstream change into types.go and marshal.go") + }) + } +} + +type fieldInfo struct{ name, typ, opts string } + +// jsonFields maps the json tag name of every serialized field to its name, type and tag +// options. The options are part of the contract: a field that lost omitempty is serialized +// where the apiserver omits it, and the desired spec could never equal the stored one again. +func jsonFields(t reflect.Type) map[string]fieldInfo { + out := make(map[string]fieldInfo, t.NumField()) + + for i := range t.NumField() { + field := t.Field(i) + + tag, opts, _ := strings.Cut(field.Tag.Get("json"), ",") + if tag == "" || tag == "-" { + continue + } + + out[tag] = fieldInfo{name: field.Name, typ: field.Type.String(), opts: opts} + } + + return out +} + +// structFields maps every field name to its type, json:"-" ones included. +func structFields(t reflect.Type) map[string]string { + out := make(map[string]string, t.NumField()) + + for i := range t.NumField() { + out[t.Field(i).Name] = t.Field(i).Type.String() + } + + return out +} + +// forkStructFields is structFields with every type rendered as the fork must declare it. +func forkStructFields(t reflect.Type) map[string]string { + out := structFields(t) + + for name, typ := range out { + out[name] = forkType(typ) + } + + return out +} + +// forkTypes are the upstream types this package retargets at itself. Every other type a +// field can hold — v1.JSON, v1.JSONSchemaURL, v1.ValidationRules — must stay upstream. +var forkTypes = []string{ + "JSONSchemaProps", + "JSONSchemaPropsOrArray", + "JSONSchemaPropsOrBool", + "JSONSchemaPropsOrStringArray", + "JSONSchemaDependencies", + "JSONSchemaDefinitions", +} + +// forkType renders an upstream field type as the fork must declare it. The mapping runs in +// this direction on purpose: erasing the package name on both sides instead would let a +// nested schema position left pointing at apiextensionsv1 compare equal — and that is the +// one drift these guards exist to catch, because Prune would then silently strip +// x-kubernetes-sensitive-data from every schema under it. +func forkType(upstream string) string { + for _, name := range forkTypes { + upstream = strings.ReplaceAll(upstream, "v1."+name, "openapi."+name) + } + + return upstream +} + +// roundTrip runs a schema through the fork the same way the installer does. +func roundTrip(t *testing.T, in map[string]any) map[string]any { + t.Helper() + + out, err := Prune(in) + require.NoError(t, err) + + return out +} + +func TestRoundTripKeepsKnownFields(t *testing.T) { + in := map[string]any{ + "type": "object", + "x-kubernetes-preserve-unknown-fields": true, + "required": []any{"name"}, + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + "maxLength": int64(10), + "enum": []any{"a", "b"}, + "default": "a", + }, + "token": map[string]any{ + "type": "string", + "x-kubernetes-sensitive-data": true, + }, + "port": map[string]any{ + "type": "integer", + "minimum": int64(1), + "maximum": int64(65535), + "multipleOf": int64(2), + }, + "ratio": map[string]any{ + "type": "number", + "maximum": 1.5, + }, + // items in single-schema form + "tags": map[string]any{ + "type": "array", + "x-kubernetes-list-type": "set", + "items": map[string]any{ + "type": "string", + "minLength": int64(1), + }, + }, + // items in tuple form + "pair": map[string]any{ + "type": "array", + "items": []any{ + map[string]any{"type": "string"}, + map[string]any{"type": "integer"}, + }, + }, + // additionalProperties in schema form + "labels": map[string]any{ + "type": "object", + "additionalProperties": map[string]any{ + "type": "string", + "x-kubernetes-sensitive-data": true, + }, + }, + // additionalProperties in bool form + "free": map[string]any{ + "type": "object", + "additionalProperties": true, + }, + "either": map[string]any{ + "allOf": []any{ + map[string]any{"type": "string"}, + map[string]any{"minLength": int64(2)}, + }, + }, + }, + } + + out := roundTrip(t, in) + + assert.Equal(t, "object", out["type"]) + assert.Equal(t, true, out["x-kubernetes-preserve-unknown-fields"]) + assert.Equal(t, []any{"name"}, out["required"]) + + props := out["properties"].(map[string]any) + + name := props["name"].(map[string]any) + // Numbers must come back as int64. A plain encoding/json round trip would yield + // float64 here, which never compares equal to what the apiserver returns and would + // make the installer issue an Update on every run. + assert.Equal(t, int64(10), name["maxLength"]) + assert.Equal(t, []any{"a", "b"}, name["enum"]) + assert.Equal(t, "a", name["default"]) + + assert.Equal(t, true, props["token"].(map[string]any)["x-kubernetes-sensitive-data"]) + + // The bounds are *float64 in Go but whole numbers on the wire, and the apiserver + // returns them as int64. Encoding them as float64 would make the desired spec + // permanently differ from the stored one and update every such CRD on every run. + port := props["port"].(map[string]any) + assert.Equal(t, int64(1), port["minimum"]) + assert.Equal(t, int64(65535), port["maximum"]) + assert.Equal(t, int64(2), port["multipleOf"]) + + assert.Equal(t, 1.5, props["ratio"].(map[string]any)["maximum"], "a fractional bound stays a float") + + tags := props["tags"].(map[string]any) + assert.Equal(t, "set", tags["x-kubernetes-list-type"]) + assert.Equal(t, int64(1), tags["items"].(map[string]any)["minLength"]) + + pair := props["pair"].(map[string]any)["items"].([]any) + require.Len(t, pair, 2) + assert.Equal(t, "integer", pair[1].(map[string]any)["type"]) + + labels := props["labels"].(map[string]any)["additionalProperties"].(map[string]any) + assert.Equal(t, true, labels["x-kubernetes-sensitive-data"], + "the extension must survive inside additionalProperties too") + + assert.Equal(t, true, props["free"].(map[string]any)["additionalProperties"]) + + either := props["either"].(map[string]any)["allOf"].([]any) + require.Len(t, either, 2) + assert.Equal(t, int64(2), either[1].(map[string]any)["minLength"]) +} + +func TestRoundTripDropsUnknownFields(t *testing.T) { + in := map[string]any{ + "type": "object", + "x-doc-examples": []any{"root"}, + "x-description": "root", + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + "x-doc-examples": []any{"nested"}, + "x-doc-default": "nested", + "x-examples": []any{"nested"}, + "x-kubernetes-immutable": true, + "x-kubernetes-patch-strategy": "merge", + }, + "tags": map[string]any{ + "type": "array", + "items": map[string]any{ + "type": "string", + "x-doc-examples": []any{"in items"}, + }, + }, + "labels": map[string]any{ + "type": "object", + "additionalProperties": map[string]any{ + "type": "string", + "x-doc-examples": []any{"in additionalProperties"}, + }, + }, + }, + } + + out := roundTrip(t, in) + + assert.NotContains(t, out, "x-doc-examples") + assert.NotContains(t, out, "x-description") + + props := out["properties"].(map[string]any) + + name := props["name"].(map[string]any) + assert.Equal(t, "string", name["type"], "known fields must survive alongside the dropped ones") + + for _, key := range []string{ + "x-doc-examples", "x-doc-default", "x-examples", + // these two look official but are not in JSONSchemaProps, so a + // x-kubernetes-* prefix rule would have let them through + "x-kubernetes-immutable", "x-kubernetes-patch-strategy", + } { + assert.NotContains(t, name, key) + } + + assert.NotContains(t, props["tags"].(map[string]any)["items"].(map[string]any), "x-doc-examples") + assert.NotContains(t, props["labels"].(map[string]any)["additionalProperties"].(map[string]any), "x-doc-examples") +} diff --git a/pkg/crd-installer/testdata/10_multi_document.yaml b/pkg/crd-installer/testdata/10_multi_document.yaml new file mode 100644 index 00000000..140d5ae6 --- /dev/null +++ b/pkg/crd-installer/testdata/10_multi_document.yaml @@ -0,0 +1,65 @@ +# Three CRDs in one file, the middle one broken: x-kubernetes-sensitive-data is a bool, +# so the schema cannot be decoded. The other two must still be installed. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: firsts.example.com +spec: + group: example.com + names: + kind: First + listKind: FirstList + plural: firsts + singular: first + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: broken.example.com +spec: + group: example.com + names: + kind: Broken + listKind: BrokenList + plural: broken + singular: broken + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + token: + type: string + x-kubernetes-sensitive-data: "yes" +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: lasts.example.com +spec: + group: example.com + names: + kind: Last + listKind: LastList + plural: lasts + singular: last + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/crd-installer/testdata/6_unknown_extensions.yaml b/pkg/crd-installer/testdata/6_unknown_extensions.yaml new file mode 100644 index 00000000..d5ea40cf --- /dev/null +++ b/pkg/crd-installer/testdata/6_unknown_extensions.yaml @@ -0,0 +1,31 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: extensions.example.com +spec: + group: example.com + names: + kind: Extension + listKind: ExtensionList + plural: extensions + singular: extension + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-doc-examples: + - at the root + properties: + spec: + type: object + properties: + token: + type: string + # the one extension that must survive, next to one that must not + x-kubernetes-sensitive-data: true + x-doc-examples: + - deep in the tree diff --git a/pkg/crd-installer/testdata/7_churn_stored.yaml b/pkg/crd-installer/testdata/7_churn_stored.yaml new file mode 100644 index 00000000..084b635a --- /dev/null +++ b/pkg/crd-installer/testdata/7_churn_stored.yaml @@ -0,0 +1,43 @@ +# What the apiserver ends up storing for 8_churn_manifest.yaml: the x-doc-* keys pruned +# away and spec.names.singular/listKind defaulted in. Installing this first puts the fake +# client into the state a real cluster would be in, which is the only way to reproduce the +# reconcile churn. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: churns.example.com +spec: + group: example.com + names: + kind: Churn + listKind: ChurnList + plural: churns + singular: churn + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + token: + type: string + x-kubernetes-sensitive-data: true + port: + type: integer + minimum: 1 + maximum: 65535 + multipleOf: 2 + - name: v1alpha1 + served: true + # the manifest omits this one: served and storage have no omitempty upstream, so the + # stored object carries both on every version no matter what the manifest says + storage: false + schema: + openAPIV3Schema: + type: object diff --git a/pkg/crd-installer/testdata/8_churn_manifest.yaml b/pkg/crd-installer/testdata/8_churn_manifest.yaml new file mode 100644 index 00000000..4cb3336d --- /dev/null +++ b/pkg/crd-installer/testdata/8_churn_manifest.yaml @@ -0,0 +1,45 @@ +# The CRD as a module actually ships it: x-doc-* keys the apiserver prunes, whole-number +# bounds it returns as int64, and no spec.names.singular/listKind because it defaults +# those itself. Applying this over 7_churn_stored.yaml must be a no-op — there is nothing +# the cluster does not already have. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: churns.example.com +spec: + group: example.com + names: + kind: Churn + plural: churns + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-doc-examples: + - at the root + properties: + spec: + type: object + properties: + token: + type: string + x-kubernetes-sensitive-data: true + x-doc-default: "" + x-doc-examples: + - a token + port: + type: integer + minimum: 1 + maximum: 65535 + multipleOf: 2 + x-doc-examples: + - 8080 + - name: v1alpha1 + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/crd-installer/testdata/9_unknown_crd_field.yaml b/pkg/crd-installer/testdata/9_unknown_crd_field.yaml new file mode 100644 index 00000000..009ddd9d --- /dev/null +++ b/pkg/crd-installer/testdata/9_unknown_crd_field.yaml @@ -0,0 +1,23 @@ +# A CRD carrying a field outside the schema that this build's apiextensions-apiserver +# does not model — a newer Kubernetes, or a Deckhouse apiserver patch. It must reach the +# cluster untouched: only schemas are pruned. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: futures.example.com +spec: + group: example.com + names: + kind: Future + listKind: FutureList + plural: futures + singular: future + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + fieldFromTheFuture: a field from a newer apiserver + schema: + openAPIV3Schema: + type: object