-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathplan_flush.go
More file actions
1838 lines (1737 loc) · 80 KB
/
Copy pathplan_flush.go
File metadata and controls
1838 lines (1737 loc) · 80 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
package git
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"os"
"path"
"path/filepath"
"sort"
"strings"
gogit "github.com/go-git/go-git/v6"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/log"
sigsyaml "sigs.k8s.io/yaml"
v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3"
"github.com/ConfigButler/gitops-reverser/internal/git/manifestedit"
"github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer"
"github.com/ConfigButler/gitops-reverser/internal/manifestreport"
"github.com/ConfigButler/gitops-reverser/internal/types"
"github.com/ConfigButler/gitops-reverser/internal/typeset"
)
// flushEventsToWorktree is the plan-then-flush write path: model the subtree once, resolve each
// event to a single-identity action, apply to commit-scoped buffers, flush only what changed.
//
// Every event is a single-identity intent and the writer NEVER mark-and-sweeps a batch.
// Whole-folder mark-and-sweep is the resync path, not steady state.
// mapperForCluster returns the GVK->GVR lookup for a source cluster: the per-cluster registry
// when a cluster is named and a cluster resolver is wired, else the default (local) mapper.
// The CLI and tests leave clusterMapper nil, so they always resolve against `mapper`.
func (w *BranchWorker) mapperForCluster(clusterID string) typeset.Lookup {
if clusterID != "" && w.clusterMapper != nil {
if lk := w.clusterMapper(clusterID); lk != nil {
return lk
}
}
return w.mapper
}
// clusterIDForEvents returns the source cluster the events in one base belong to. Events in a
// single flush share a GitTarget (they are grouped by base), so they share a cluster; the
// first non-empty id wins, and an all-empty set is the local cluster.
func clusterIDForEvents(events []Event) string {
for _, ev := range events {
if ev.SourceCluster != "" {
return ev.SourceCluster
}
}
return ""
}
func (w *BranchWorker) flushEventsToWorktree(
ctx context.Context,
worktree *gogit.Worktree,
base string,
events []Event,
policy *manifestanalyzer.PlacementPolicy,
namespaces namespacePolicy,
pruneMode v1alpha3.PruneMode,
) (bool, error) {
root := worktree.Filesystem().Root()
scoped, err := scanRenderScope(root, base)
if err != nil {
return false, err
}
// Every event in a base shares one GitTarget (events are grouped by base), so they share
// one source cluster; resolve this subtree's GVK->GVR against that cluster's registry.
mapper := w.mapperForCluster(clusterIDForEvents(events))
batch := newWriteBatch(ctx, w.contentWriter, mapper, scoped.scan, policy, namespaces, scoped.writeSubdir)
batch.pruneMode = pruneMode
batch.target = placementTargetForEvents(events)
if err := batch.refusal(); err != nil {
return false, err
}
// Publish what this folder's shape resolved to before folding anything in. It is a fact about
// the scan, so it must not depend on the events applying — or on their being applied at all,
// which is what makes it available to a suspended target. It runs AFTER the acceptance gate
// above on purpose: a folder the operator has refused to manage is one whose layout it should
// not be making claims about, and GitPathAccepted=False already says why.
w.scanLayout(ctx, batch, worktree)
// The one write-plan precondition that is not about the folder at all, which is why it is
// raised AFTER the layout is published: the folder is fine and its shape is still worth
// reporting; what is wrong is the configuration pointed at it.
if err := batch.sourceNamespaceRefusal(); err != nil {
return false, err
}
for _, event := range events {
if err := batch.applyEvent(ctx, event); err != nil {
return false, err
}
}
// The flush is anchored at renderBase — spec.path, or the common ancestor of spec.path
// and every base it reads. The write jail (writeSubdir) is enforced inside the batch, so
// a planned write outside spec.path is refused even though the scan reached past it.
return batch.flush(ctx, worktree, root, scoped.renderBase)
}
// writeBatch is the commit-scoped plan-then-flush working set for one GitTarget
// subtree. The store is the byte-free model the batch resolves identities against;
// contentByPath holds the worktree bytes so a touched file is hydrated lazily into
// a fileBuffer; buffers accumulates the mutations the events produce.
type writeBatch struct {
writer eventContentWriter
mapper typeset.Lookup
store *manifestanalyzer.ManifestStore
docLoc map[*manifestanalyzer.DocumentModel]manifestanalyzer.RecordRef
contentByPath map[string][]byte
buffers map[string]*fileBuffer
// intents records what each document this flush writes must render to, so the
// render precondition can tell a change the flush MEANT from one it merely caused.
// Anything not named here has to come out of the re-render untouched.
intents []manifestanalyzer.WriteIntent
// putToKustomize records that this flush touched a kustomize render root — it edited a
// governed document, or placed a new one into a kustomization's resources:. It is what
// turns the oracle on, and it is deliberately NOT the same question as WriteIntent.Governed:
// that one additionally ASSERTS the document is rendered, which a new document is not
// entitled to claim (its resources: entry can legitimately fail to be added — see
// appendKustomizationResource — leaving the file written but outside every render).
putToKustomize bool
// target is the GitTarget this batch writes for, carried only so the placement metrics
// can name it (see placement_metrics.go). It is set by the caller — the live path reads
// it off the events, the resync path from the request — and is empty for the CLI and for
// tests, where the counters are simply unlabelled.
target placementTarget
// documents is the write-boundary census for this batch, tallied as documents are decided and
// published by flush once the bytes are actually on disk. See document_metrics.go for why it
// is not published at the decision site.
documents map[documentKey]int64
// policy is the GitTarget's declared new-file placement policy, consulted
// only for a resource with no existing document. nil means no declared policy —
// placement falls through to the folder's one kustomize root and then the canonical path.
policy *manifestanalyzer.PlacementPolicy
// namespaces is the GitTarget's declared namespace behavior — spec.serializeNamespace — which
// decides whether the bytes this batch writes carry metadata.namespace at all. The zero value
// is "declare nothing", i.e. infer per document, which is what every caller with no GitTarget
// to read (the CLI, most tests) gets.
namespaces namespacePolicy
// pruneMode gates the EXPLICIT delete path only; inferred mark-and-sweep is gated in the
// planner. Always read through OrDefault: the zero value is unset, not `never`, and the resync
// batch never sets it — reading it literally would stop that batch mirroring deletes.
pruneMode v1alpha3.PruneMode
// writeSubdir is spec.path expressed relative to the render anchor (renderBase) — the
// write jail. It is "" for a self-contained subtree (renderBase == spec.path), where
// every scanned path is writable; it is non-empty only when the scan reached past
// spec.path into a base it renders (render-root scoping), and then a planned write must
// stay within it. The store and every path in it are keyed relative to renderBase, so a
// writable path is one under writeSubdir. See internal/git/render_scope.go.
writeSubdir string
// createdRoot is the kustomization.yaml this batch WROTE, for a folder that had none and a
// target that declared spec.placement.useKustomize. It is nil in every other case, including
// the ordinary one where a root was already there. It exists so the second new document in a
// batch joins the root the first one created: the store was built before the batch, so nothing
// in it knows the file exists.
createdRoot *manifestanalyzer.KustomizationInfo
// layout is what the scan resolved about this folder's shape. It is published as
// status.placement, and createNew reads it: a folder covering several render roots has no
// single one to place a new document into, so placing one is refused rather than guessed.
layout manifestanalyzer.LayoutResolution
// coldBundles tracks new resources placed at a path that held nothing before the batch.
// LocateNew resolves against the PRE-BATCH store, so it cannot see two new resources landing
// on one path; without this each write would silently discard the last. Members are re-sorted
// by resource identity so the result does not depend on event order.
// See docs/layout/new-file-placement-rules.md, "Collision and append behavior".
coldBundles map[string][]coldBundleMember
}
// coldBundleMember is one new document contributing to a brand-new shared bundle
// file within this batch. Retained (rather than re-parsed from buf.current) so a
// later collision on the same path can re-sort and rebuild the whole file from
// scratch, independent of which new resource's event the writer processed first.
type coldBundleMember struct {
identifier types.ResourceIdentifier
content []byte
// sensitive records whether this member is an encrypted (sensitive) resource, so
// createNew can refuse to co-mingle sensitive and plaintext documents in one
// brand-new file regardless of the order their events arrived (Option B2's
// write-safety guard — see createNew).
sensitive bool
}
func newWriteBatch(
ctx context.Context,
writer eventContentWriter,
mapper typeset.Lookup,
scan manifestanalyzer.FolderScan,
policy *manifestanalyzer.PlacementPolicy,
namespaces namespacePolicy,
writeSubdir string,
) *writeBatch {
// The writer allowlist retains build directives (kustomization.yaml) and the operator's
// own .sops.yaml bootstrap config outside the managed model — these are auxiliary input,
// not documents to materialise or to mis-refuse as standalone non-KRM. Every other KRM
// document is still materialised: the live writer indexes the whole subtree for
// placement. The scan also carries the foreign-content view and the active
// .gittargetignore, so the structure-only acceptance gate (run by writeBatch.refusal) and
// the write-plan precondition (run by writeBatch.flush) read both from the store.
store := manifestanalyzer.BuildStoreFromScan(ctx, scan, mapper, manifestanalyzer.WriterAllowlist(),
manifestanalyzer.WithDeclaredNamespace(namespaces.declaredNamespace()))
// Surface the store's build-time warnings (ambiguous namespace or override
// context, scope mismatches) once per batch: these drive silent fallbacks —
// e.g. an ambiguous override chain falls back to write-through — and without
// this line the live path would leave no trace of why. The analyzer CLI and
// scan mode show the same diagnostics offline.
logStoreDiagnostics(ctx, store.Diagnostics)
contentByPath := make(map[string][]byte, len(scan.YAMLFiles))
for _, f := range scan.YAMLFiles {
contentByPath[f.Path] = f.Content
}
batch := &writeBatch{
writer: writer,
mapper: mapper,
store: store,
docLoc: store.DocumentLocations(),
contentByPath: contentByPath,
buffers: map[string]*fileBuffer{},
policy: policy,
namespaces: namespaces,
writeSubdir: writeSubdir,
}
// Resolved with the store rather than by each caller, so no write path can reach createNew
// with an unresolved layout and place a new document into a folder that has no single root
// to place it in. Publishing it is a separate step (scanLayout), because only the paths that
// know which GitTarget they serve can publish.
batch.layout = manifestanalyzer.ResolveLayout(store, writeSubdir)
return batch
}
// refusal runs the acceptance gate over the batch's store; a refusal aborts the commit before any
// file is touched.
//
// Structure-only on purpose: refusing on a discovery-derived fact (unwatched / out-of-scope) would
// turn a discovery wobble into a stuck, unwritable GitTarget.
func (wb *writeBatch) refusal() error {
return manifestanalyzer.RefusalError(manifestanalyzer.AcceptStructureOnly(wb.store))
}
// sourceNamespaceRefusal is the write-plan precondition for the one-source-namespace rule.
//
// This is the CORRECTNESS layer and it holds whatever admission did: the WatchRule webhook is
// one-shot, cannot see a serializeNamespace flipped afterwards, and fails open.
// See docs/spec/where-validation-lives.md.
func (wb *writeBatch) sourceNamespaceRefusal() error {
issues := manifestanalyzer.MultipleSourceNamespacesRefusal(
wb.namespaces.declaresNamespaceFree(),
wb.namespaces.SourceNamespaces,
wb.namespaces.SourceNamespaceWildcard,
wb.writeSubdir,
)
if len(issues) == 0 {
return nil
}
return manifestanalyzer.RefusalError(manifestanalyzer.Acceptance{Accepted: false, Issues: issues})
}
// fileBuffer is the commit-scoped, hydrated working copy of one file under the
// GitTarget base path. original is the worktree bytes (nil for a file the batch
// creates); current is the bytes after applying actions (nil means the file should
// be removed). Dirty/Deleted are derived exactly as the design's FileModel — two
// byte slices are the whole state machine, so there is no flag to forget to flip.
type fileBuffer struct {
rel string
original []byte
current []byte
}
func (b *fileBuffer) dirty() bool { return b.current != nil && !bytes.Equal(b.current, b.original) }
func (b *fileBuffer) deleted() bool { return b.current == nil && b.original != nil }
// buffer returns the hydrated working copy for a base-relative path, reading the
// worktree bytes into Original/Current on first touch. A path with no worktree
// bytes is a new file (Original nil).
func (wb *writeBatch) buffer(rel string) *fileBuffer {
if b, ok := wb.buffers[rel]; ok {
return b
}
b := &fileBuffer{rel: rel}
if orig, ok := wb.contentByPath[rel]; ok {
b.original = orig
b.current = orig
}
wb.buffers[rel] = b
return b
}
// upsertOutcome is what an upsert actually did to the worktree bytes, so a caller can
// count create/update accurately from the apply rather than from a separate plan
// estimate (which mislabels a re-encrypted sensitive resource as skipped).
type upsertOutcome int
const (
upsertNoChange upsertOutcome = iota
upsertCreated
upsertUpdated
// upsertSkippedUnsafe is a deliberate, fail-safe refusal to write a resource:
// its placement could not be resolved safely, or writing would co-mingle a
// sensitive and a plaintext document, or would overwrite a multi-document file.
// It is distinct from upsertNoChange (a genuine no-op) so the resync path can
// count it and surface it, rather than have a not-mirrored resource vanish with
// no signal (placement Option B2's fail-safe skips — see createNew/writeWholeFile).
upsertSkippedUnsafe
)
// applyEvent folds one event into the batch: a field patch sets bounded fields on an
// existing parent, a DELETE removes a document, anything else is an upsert (the
// object-bearing event the stream guarantees for non-deletes). The steady-state
// writer does not need the upsert outcome (it flushes by byte state), so it is
// discarded here; the resync planner consumes it for stats.
func (wb *writeBatch) applyEvent(ctx context.Context, event Event) error {
switch {
case event.IsFieldPatch():
return wb.applyFieldPatch(ctx, event)
case event.Operation == "DELETE":
wb.applyDelete(ctx, event)
return nil
default:
_, err := wb.applyUpsert(ctx, event)
return err
}
}
// applyUpsert resolves an object-bearing event against the subtree. When a managed
// document for its identity already lives there — even moved off the canonical path —
// the resource is edited where it lives: a non-sensitive document is patched in place;
// a sensitive document is re-encrypted wholesale AT ITS EXISTING PATH (never patched in
// place — that would drop the SOPS metadata and write the secret back in cleartext, and
// never at the canonical path, which would orphan the moved copy). A resource with no
// existing document is placed by createNew. It returns what it did to the bytes
// (created / updated / no change).
func (wb *writeBatch) applyUpsert(ctx context.Context, event Event) (upsertOutcome, error) {
outcome, err := wb.upsert(ctx, event)
if err == nil {
// Tallied here rather than at either caller: the live path reaches this through applyEvent
// and the resync path calls it directly, so this is the one place both are covered exactly
// once. Published by flush, never here — see document_metrics.go.
wb.tallyDocument(event.Identifier, documentOutcomeForUpsert(outcome))
}
return outcome, err
}
// upsert is applyUpsert's body, split out so the census above wraps every return path.
func (wb *writeBatch) upsert(ctx context.Context, event Event) (upsertOutcome, error) {
id, ok := manifestIdentity(event.Object)
if !ok {
return wb.createNew(ctx, event)
}
dm := wb.store.ByManifestIdentity[id]
if dm == nil {
return wb.createNew(ctx, event)
}
filePath := wb.docLoc[dm].FilePath
if !wb.writer.isSensitiveIdentifier(event.Identifier) {
return wb.patchExisting(ctx, event, filePath, id, dm)
}
return wb.rewriteSensitive(ctx, event, filePath)
}
// rewriteSensitive re-encrypts a sensitive document wholesale at its existing path.
//
// Its intent is UNCHECKED: the file is SOPS ciphertext, so kustomize renders the encrypted
// blob and no plaintext live object can ever equal it. The oracle is told to expect this
// object to move without being able to say what to — while still holding the write to
// disturbing nothing else, which is the half that protects other environments.
func (wb *writeBatch) rewriteSensitive(ctx context.Context, event Event, filePath string) (upsertOutcome, error) {
outcome, err := wb.writeWholeFile(ctx, event, filePath)
if err == nil && wroteBytes(outcome) {
wb.intend(markUnchecked(intentFor(event.Object, filePath, false), true))
}
return outcome, err
}
// wroteBytes reports whether an upsert actually changed the worktree, which is the only
// case that owes the oracle an intent.
func wroteBytes(o upsertOutcome) bool {
return o == upsertCreated || o == upsertUpdated
}
// createNew places a resource with no existing document, per
// docs/layout/new-file-placement-rules.md, and adds any kustomize resources: entry it requires.
// A placement that cannot be honoured safely (a sensitive resource colliding with an existing
// file) is logged and left unwritten rather than mis-written; the next event or resync retries.
func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome, error) {
kind := ""
if event.Object != nil {
kind = event.Object.GetKind()
}
// A folder covering several render roots has no single one for a NEW document, and picking
// one would hand it to an environment nobody named. An existing document is unaffected: it
// is edited where it lives, and this branch is not reached for it.
if issues := manifestanalyzer.AmbiguousLayoutRefusal(wb.layout, wb.writeSubdir); len(issues) > 0 {
return upsertNoChange, manifestanalyzer.RefusalError(
manifestanalyzer.Acceptance{Accepted: false, Issues: issues})
}
sensitive := wb.writer.isSensitiveIdentifier(event.Identifier)
// WriteScope tells placement the write jail: when render-root scoping re-rooted the scan
// past spec.path, a declared/canonical path is rebased under the jail rather than escaping
// it (see finishPlacement). It is "" for a self-contained subtree, where placement already
// resolves relative to spec.path.
placement, err := manifestanalyzer.LocateNew(wb.store, wb.policy, manifestanalyzer.PlacementRequest{
Identifier: event.Identifier,
Kind: kind,
Sensitive: sensitive,
WriteScope: wb.writeSubdir,
})
if err != nil {
refusal := placementRefusalReason(err)
log.FromContext(ctx).Info("Skipping new resource: placement could not be resolved safely",
"resource", event.Identifier.String(), "refusal", string(refusal), "reason", err.Error())
recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal)
return upsertSkippedUnsafe, nil
}
// The LIVE object, kept before the namespace strip below rewrites it. The bytes we write
// and the object the render must produce are not the same thing, and only this scope
// still holds both — see intentFor.
live := event.Object
// A destination that infers its namespace from build context (a kustomization's
// namespace: transformer) must keep metadata.namespace out of the written bytes,
// exactly as patchExisting already does for an in-place edit of an existing
// document in the same context — otherwise the new document would silently break
// the convention every sibling in that directory follows. An explicit
// spec.serializeNamespace overrides that inference in both directions.
if wb.namespaces.omitNamespace(placement.NamespaceInherited) && event.Object != nil {
event.Object = event.Object.DeepCopy()
event.Object.SetNamespace("")
}
outcome, refusal, err := wb.placeNewDocument(ctx, event, placement, sensitive)
if err != nil || !wroteBytes(outcome) {
// A skipped write is a resource the mirror does not hold. Count it with the refusals
// LocateNew raised, from the same closed reason set, so "resources we declined to
// place" is one series rather than a log line here and a metric there.
if outcome == upsertSkippedUnsafe {
recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal)
}
return outcome, err
}
// Recorded here rather than at resolution: this is the point at which the document is
// really in the mirror at this path, so placements_total and placement_refusals_total
// partition every new resource instead of double-counting the ones that resolved and
// then could not be written.
recordPlacement(ctx, wb.target, event.Identifier, placement.Source, placement.Append)
// AFTER the write, for the same reason the placement is counted here. placeNewDocument can
// still decline — a multi-document target it will not overwrite, or a new file that would
// mix sensitive and plaintext documents — and registering the entry first meant a resource we
// REFUSED still put its file into the folder's render. For the multi-document case that is
// foreign content we declined to own, added to resources: on our say-so; and either way it
// counted as outcome="added", the value that is supposed to mean "the file we just wrote will
// build". Pinned by TestPlacementMetrics_RefusedPlacementLeavesTheKustomizationAlone.
if err := wb.resolveKustomizeRootForNew(ctx, &placement); err != nil {
return upsertNoChange, err
}
if placement.Kustomization != nil {
wb.appendKustomizationResource(ctx, event, placement)
}
// A new document inside a render root is governed by the folder's images:/replicas: entries
// immediately, but has no override chain to route onto, so an entry that overrides it makes
// the resource never converge. Declaring it governed puts it in front of the oracle, turning a
// silent non-converging commit into a refusal naming the file and object.
wb.putToKustomize = wb.putToKustomize || placement.Kustomization != nil || wb.createdRoot != nil
wb.intend(markUnchecked(intentFor(live, placement.Path, false), sensitive))
return outcome, nil
}
// resolveKustomizeRootForNew settles what renders a new document, for a target that declared
// spec.placement.useKustomize. It creates the folder's root when there is none — with this document
// already registered in the bytes it writes, so a LATER document in the same batch joins it through
// the ordinary resources: append — and otherwise refuses a placement no kustomization would render.
//
// Both halves are here rather than in createNew because they answer one question between them: is
// this document going to be rendered at all?
func (wb *writeBatch) resolveKustomizeRootForNew(
ctx context.Context,
placement *manifestanalyzer.PlacementResult,
) error {
if created := wb.bootstrapKustomization(ctx, *placement); created != nil {
placement.Kustomization = created
}
issues := wb.unrenderedPlacementRefusal(*placement)
if len(issues) == 0 {
return nil
}
return manifestanalyzer.RefusalError(manifestanalyzer.Acceptance{Accepted: false, Issues: issues})
}
// unrenderedPlacementRefusal reports the refusal for a placement under spec.placement.useKustomize
// that no kustomization would render. A document is rendered when a kustomization governs its path,
// and there are three ways for that to be true: one already governed it and needs the entry
// (LocateNew set Kustomization), one already governed it and already lists the path, or this batch
// created the root — which registers the first document in the bytes it writes, so it reports no
// Kustomization to append to.
func (wb *writeBatch) unrenderedPlacementRefusal(
placement manifestanalyzer.PlacementResult,
) []manifestanalyzer.AcceptanceIssue {
governed := placement.Kustomization != nil ||
wb.createdRoot != nil ||
manifestanalyzer.GoverningKustomization(wb.store, wb.writeSubdir, placement.Path) != nil
useKustomize := wb.policy != nil && wb.policy.UseKustomize
return manifestanalyzer.UnrenderedPlacementRefusal(
useKustomize, governed, placement.Path, wb.layout.RenderRoot)
}
// placeNewDocument writes the new document at its resolved placement: appended to an existing
// accepted bundle, folded into a same-batch cold bundle, or as a file of its own.
//
// It returns the refusal reason alongside the outcome, and only for upsertSkippedUnsafe, so
// the caller can count WHY a new resource was left out of the mirror without inspecting a log
// message. The multi-document refusal is attributed here rather than inside writeWholeFile
// because that function also serves in-place updates, where the same skip is not a placement
// decision at all.
func (wb *writeBatch) placeNewDocument(
ctx context.Context,
event Event,
placement manifestanalyzer.PlacementResult,
sensitive bool,
) (upsertOutcome, manifestanalyzer.PlacementRefusalReason, error) {
if placement.Append {
outcome, err := wb.appendNewDocument(ctx, event, placement.Path)
return outcome, "", err
}
buf := wb.buffer(placement.Path)
if buf.original == nil {
// Nothing occupied this path pre-batch, so route through the cold-bundle path: a collision
// LocateNew could not see must form a deterministic multi-document file rather than one
// write discarding another.
//
// A sensitive resource never shares a file, and plaintext never joins a bundle holding a
// sensitive member. Skip rather than co-mingle; the next event or resync retries.
if buf.current != nil && (sensitive || wb.coldBundleHasSensitive(placement.Path)) {
log.FromContext(ctx).Info(
"Skipping new resource: sensitive and plaintext resources must not share a new file",
"resource", event.Identifier.String(), "file", placement.Path, "sensitive", sensitive)
return upsertSkippedUnsafe, manifestanalyzer.PlacementRefusedMixedSensitivityNewFile, nil
}
outcome, err := wb.writeColdBundleMember(ctx, event, placement.Path, sensitive)
return outcome, "", err
}
outcome, err := wb.writeWholeFile(ctx, event, placement.Path)
if outcome == upsertSkippedUnsafe {
return outcome, manifestanalyzer.PlacementRefusedMultiDocumentTarget, err
}
return outcome, "", err
}
// writeColdBundleMember writes a resource to a path nothing occupied before this batch. Every
// member seen at rel is re-sorted by resource identity and the file rebuilt, so the result does
// not depend on which event was processed first. The single-member case is byte-identical to a
// plain write.
func (wb *writeBatch) writeColdBundleMember(
ctx context.Context,
event Event,
rel string,
sensitive bool,
) (upsertOutcome, error) {
content, err := wb.writer.buildContentForWrite(ctx, event)
if err != nil {
return upsertNoChange, err
}
if wb.coldBundles == nil {
wb.coldBundles = map[string][]coldBundleMember{}
}
wb.coldBundles[rel] = append(
wb.coldBundles[rel],
coldBundleMember{identifier: event.Identifier, content: content, sensitive: sensitive},
)
members := wb.coldBundles[rel]
sort.Slice(members, func(i, j int) bool {
return members[i].identifier.Key() < members[j].identifier.Key()
})
var rebuilt []byte
for _, m := range members {
rebuilt = appendYAMLDocument(rebuilt, m.content)
}
wb.buffer(rel).current = rebuilt
return upsertCreated, nil
}
// coldBundleHasSensitive reports whether any member already staged for the
// brand-new file at rel is an encrypted (sensitive) resource, so createNew can
// refuse to add a plaintext member that would co-mingle with it.
func (wb *writeBatch) coldBundleHasSensitive(rel string) bool {
for _, m := range wb.coldBundles[rel] {
if m.sensitive {
return true
}
}
return false
}
// appendNewDocument adds a resource with no existing document as an additional
// document in an existing accepted plaintext file (a "bundle" placement). Unlike
// writeWholeFile it never replaces the file's existing bytes — every prior document
// in the buffer survives untouched, byte for byte; LocateNew never returns an
// Append placement for a sensitive resource (see its doc comment), so this path is
// plaintext-only.
func (wb *writeBatch) appendNewDocument(ctx context.Context, event Event, rel string) (upsertOutcome, error) {
content, err := wb.writer.buildContentForWrite(ctx, event)
if err != nil {
return upsertNoChange, err
}
buf := wb.buffer(rel)
buf.current = appendYAMLDocument(buf.current, content)
return upsertCreated, nil
}
// appendYAMLDocument appends newDoc as an additional "---\n"-separated document
// after existing. existing is assumed to already be valid, accepted YAML (single- or
// multi-document); newDoc is assumed to be exactly one well-formed document
// (sanitize.MarshalToOrderedYAML's output, which always ends in a newline).
func appendYAMLDocument(existing, newDoc []byte) []byte {
if len(existing) == 0 {
return newDoc
}
const separator = "---\n"
out := make([]byte, 0, len(existing)+len(separator)+len(newDoc))
out = append(out, existing...)
if out[len(out)-1] != '\n' {
out = append(out, '\n')
}
out = append(out, separator...)
out = append(out, newDoc...)
return out
}
// appendKustomizationResource adds the new document's path to its resources:
// sequence as part of the same commit, so kustomize picks up the file createNew just
// placed inside the kustomization's directory — the "add to the right kustomize
// file." The entry is rendered relative to the kustomization's own directory
// (resources: entries are relative to the kustomization file, not the repo root).
// A failure here only drops the resources: entry (logged as a diagnostic); the
// resource's own file is still written, since a human can add the missing entry by
// hand and the next placement for that directory re-detects the gap.
func (wb *writeBatch) appendKustomizationResource(
ctx context.Context,
event Event,
placement manifestanalyzer.PlacementResult,
) {
k := placement.Kustomization
entry := placement.Path
if dir := path.Dir(k.Path); dir != "." {
if rel, err := filepath.Rel(dir, placement.Path); err == nil {
entry = filepath.ToSlash(rel)
}
}
buf := wb.buffer(k.Path)
if buf.current == nil {
// The kustomization vanished within this batch; nothing to edit — and the file it
// would have registered is now outside every render, which is the same user-visible
// outcome as a failed edit, so it is counted as one.
recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed)
return
}
res, diags := manifestedit.AppendKustomizationResource(k.Path, buf.current, entry)
switch res.Mode {
case manifestedit.EditPatched:
buf.current = res.Content
recordKustomizationEntry(ctx, wb.target, kustomizationEntryAdded)
log.FromContext(ctx).Info("Added resources: entry for new file",
"kustomization", k.Path, "entry", entry, "resource", event.Identifier.String())
case manifestedit.EditNoChange:
recordKustomizationEntry(ctx, wb.target, kustomizationEntryNoChange)
case manifestedit.EditSkipped, manifestedit.EditDeleted, manifestedit.EditWholeReplace:
// The document is committed and its resources: entry is not, so kustomize will never
// build the file: it is in Git, it looks mirrored, and nothing applies it. The counter
// is the only signal that is not a log line.
recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed)
log.FromContext(ctx).Info("Could not add resources: entry for new file",
"kustomization", k.Path, "entry", entry, "resource", event.Identifier.String())
logManifestDiagnostics(ctx, diags)
}
}
// applyFieldPatch sets only the patch's declared field paths on the existing parent document.
//
// Two refusals make a partial intent safe:
// - NO creation path: a patch whose parent is absent is dropped rather than fabricated.
// - A document that cannot be patched field-by-field is SKIPPED, never whole-replaced, which
// would delete every field the subresource did not mention.
//
// The document index is re-derived from CURRENT bytes so an earlier event in the batch that
// shifted a multi-document file does not misdirect the edit.
func (wb *writeBatch) applyFieldPatch(ctx context.Context, event Event) error {
filePath, id, ok := wb.resolveFieldPatchTarget(event)
if !ok {
log.FromContext(ctx).Info("Dropping field patch: parent manifest not present in Git",
"resource", event.Identifier.String(), "source", event.FieldPatch.Source,
"reason", "subresource_patch_no_parent")
return nil
}
assignments := event.FieldPatch.Assignments
dm := wb.store.ByManifestIdentity[id]
governed := dm != nil && dm.Overrides != nil
if governed {
assignments = wb.routeGovernedFieldAssignments(ctx, event, dm, assignments)
// A routed scale changes only the kustomization entry, which still moves what this
// document renders to — so it must be declared, or the oracle would read its own
// intended write as collateral damage. It is UNCHECKED because a field patch carries
// a few audited assignments, never a whole object to compare the render against: the
// oracle can still prove the write disturbs nothing else, but not that it landed.
wb.putToKustomize = true
wb.intend(fieldPatchIntent(filePath, id, governed))
if len(assignments) == 0 {
return nil
}
}
buf := wb.buffer(filePath)
idx, found := currentDocIndex(filePath, buf.current, id)
if !found {
// An earlier event in this batch already removed the document; nothing to patch.
return nil
}
res, diags := manifestedit.PatchFields(
buf.current, idx, id, assignments, manifestedit.EditOptions{},
)
switch res.Mode {
case manifestedit.EditPatched:
buf.current = res.Content
if !governed {
wb.intend(fieldPatchIntent(filePath, id, false))
}
case manifestedit.EditNoChange, manifestedit.EditDeleted:
// No-op: the audited value already matched (or, impossible here, a delete).
case manifestedit.EditSkipped, manifestedit.EditWholeReplace:
// EditSkipped (encrypted, non-editable, or snapshot drift), or a defensive
// EditWholeReplace we must never apply from a partial desired.
log.FromContext(ctx).Info("Field patch not applied: parent is encrypted or not field-patchable",
"resource", event.Identifier.String(), "source", event.FieldPatch.Source,
"reason", "subresource_patch_unsafe")
logManifestDiagnostics(ctx, diags)
}
return nil
}
// routeGovernedFieldAssignments diverts a spec.replicas assignment whose value a
// replicas override governs to its kustomization entry (the /scale subresource
// case of the images/replicas edit-through) and returns the assignments the file
// patch should still apply. An
// ungoverned assignment — any other path, a non-integer value, no matching
// entry — keeps today's bounded file patch.
func (wb *writeBatch) routeGovernedFieldAssignments(
ctx context.Context,
event Event,
dm *manifestanalyzer.DocumentModel,
assignments []manifestedit.FieldAssignment,
) []manifestedit.FieldAssignment {
kept := make([]manifestedit.FieldAssignment, 0, len(assignments))
for _, a := range assignments {
if len(a.Path) == 2 && a.Path[0] == "spec" && a.Path[1] == "replicas" {
if count, isInt := assignmentInt64(a.Value); isInt {
if edit, governed := manifestanalyzer.ReplicaCountEdit(dm, count); governed {
wb.applyOverrideEdits(ctx, event, []manifestanalyzer.OverrideEdit{edit})
continue
}
}
}
kept = append(kept, a)
}
return kept
}
// assignmentInt64 reads a field-assignment value as a whole number (audit JSON
// may deliver it as int64 or float64).
func assignmentInt64(v any) (int64, bool) {
switch n := v.(type) {
case int64:
return n, true
case int:
return int64(n), true
case int32:
return int64(n), true
case float64:
if n == math.Trunc(n) {
return int64(n), true
}
}
return 0, false
}
// resolveFieldPatchTarget locates the parent manifest a field-patch event targets.
// The parent is resolved from its objectRef GVR through the same resource-identity
// inventory the GVR-only delete uses (PlanDelete), which the live-catalog mapper
// populates while scanning the GitTarget folder. The returned identity is the parent
// document's own manifest identity (full GVK from the committed YAML), so the patch
// is applied with the parent's real Kind, never one guessed from the subresource body.
//
// found is false when Git holds no managed document for the parent identity.
func (wb *writeBatch) resolveFieldPatchTarget(event Event) (string, manifestedit.Identity, bool) {
if action, emitted := manifestanalyzer.PlanDelete(wb.store, event.Identifier); emitted {
return action.Ref.FilePath, action.Identity, true
}
return "", manifestedit.Identity{}, false
}
// patchExisting edits the managed document in place, preserving sibling bytes and hand-authored
// formatting. The no-op / patch / whole-replace / skip choice is a plan decision, not a per-event
// heuristic, and the position is re-derived from CURRENT bytes so an earlier event in the batch
// does not misdirect it.
//
// Under a kustomize images/replicas override chain the desired projection is split first: values
// the chain produces are restored to source form and the divergence routed to the entries.
func (wb *writeBatch) patchExisting(
ctx context.Context,
event Event,
filePath string,
id manifestedit.Identity,
dm *manifestanalyzer.DocumentModel,
) (upsertOutcome, error) {
buf := wb.buffer(filePath)
idx, ok := currentDocIndex(filePath, buf.current, rawManifestIDForCurrentBytes(id, dm))
if !ok {
return upsertNoChange, nil
}
gitDoc, _ := manifestedit.NewDocumentAt(filePath, buf.current, idx)
desired := event.Object
if wb.namespaces.omitNamespace(dm.NamespaceAbsentFromFile()) && desired != nil {
desired = desired.DeepCopy()
desired.SetNamespace("")
}
projected, overrideEdits, err := projectThroughKustomize(
manifestreport.Project(desired), buf.current, idx, dm, wb.overlayAuthorKustomization(filePath))
if err != nil {
var fidelity *renderFidelityRefusedError
if errors.As(err, &fidelity) {
return upsertNoChange, renderFidelityRefusal(filePath, id, fidelity)
}
// The projection could not place the edit. Refusing the whole flush is the point: the
// alternative is to write the live object through and silently absorb the build's own
// output into the file that feeds it.
return upsertNoChange, sourceFormRefusal(filePath, id, err)
}
c := manifestedit.Comparison{
Git: gitDoc,
Desired: projected,
Options: manifestreport.EditOptions(),
}
res, diags := manifestedit.Apply(c, manifestedit.Decide(c))
outcome := upsertNoChange
switch res.Mode {
case manifestedit.EditPatched, manifestedit.EditWholeReplace:
buf.current = res.Content
outcome = upsertUpdated
case manifestedit.EditNoChange, manifestedit.EditSkipped, manifestedit.EditDeleted:
// No-op, an unsafe edit left untouched, or (impossible here) a delete: leave
// the bytes as they are. Surface a skip so an operator can see a document Git
// holds but the editor refused.
if res.Mode == manifestedit.EditSkipped {
logManifestDiagnostics(ctx, diags)
}
}
if wb.applyOverrideEdits(ctx, event, overrideEdits) {
outcome = upsertUpdated
}
// Declare what this document must render to. Attribution decided WHERE the edit goes and may
// be wrong; renderPrecondition adjudicates once the whole plan is known.
//
// A GOVERNED document declares intent even when its own bytes did not change. images: entries
// are shared: bumping two Deployments on one image means the first event's edit already moved
// what the second renders to, leaving nothing to write. Its render still moves, onto its own
// live state, and only the declared intent says that is convergence rather than damage.
//
// The oracle is armed for ANY document a render root produces, not only one an override chain
// governs: where live and render disagree the user changed something, and if a transformer
// owns that field the write never converges. Only the re-render can see that.
if dm.Rendered != nil {
wb.putToKustomize = true
}
if outcome == upsertUpdated || dm.Overrides != nil {
wb.intend(intentFor(event.Object, filePath, dm.Overrides != nil))
}
return outcome, nil
}
// projectThroughKustomize turns the live projection into the SOURCE FORM of it: the object the
// file should hold once everything the build supplies is left to the build, plus the entry edits
// for the values an images:/replicas: entry supplies.
//
// A plain document uses its parsed Git object as its render. A kustomize document uses the
// DocumentModel's local render. In both cases, a rendered ${...} value that differs in live is
// refused before source-form projection can write the live expansion back into Git.
func projectThroughKustomize(
projected *unstructured.Unstructured,
content []byte,
idx int,
dm *manifestanalyzer.DocumentModel,
authorInto string,
) (*unstructured.Unstructured, []manifestanalyzer.OverrideEdit, error) {
gitRaw, parsed := gitDocRawObject(content, idx)
if !parsed {
return projected, nil, nil
}
rendered := gitRaw
if dm.Rendered != nil {
rendered = dm.Rendered.Object
}
if divergences := manifestanalyzer.RenderTokenDivergences(rendered, projected.Object); len(divergences) > 0 {
return nil, nil, &renderFidelityRefusedError{Divergences: divergences}
}
if dm.Rendered == nil {
return projected, nil, nil
}
return manifestanalyzer.SplitDesiredForOverrides(gitRaw, projected, dm.Rendered, authorInto)
}
// overlayAuthorKustomization is the kustomization the writer may author a NEW images:/replicas:
// entry into for an edit to filePath. It is set only when render-root scoping put filePath OUT of
// the write jail — a base document an overlay reads read-only — and the overlay has a supported
// render root of its own: then a value the base supplies can be overridden by authoring an entry
// in the overlay instead of refusing the base write. It is "" for a self-contained subtree and
// for an in-jail document, where the source file itself is writable.
func (wb *writeBatch) overlayAuthorKustomization(filePath string) string {
if wb.writeSubdir == "" || pathWithin(filePath, wb.writeSubdir) {
return ""
}
if k := wb.store.Kustomizations[wb.writeSubdir]; k != nil && !k.Unsupported {
return k.Path
}
return ""
}
// renderFidelityRefusedError travels from the projection seam to patchExisting, where the file
// and object identity are available to make a normal write-boundary refusal.
type renderFidelityRefusedError struct {
Divergences []manifestanalyzer.RenderDivergence
}
func (e *renderFidelityRefusedError) Error() string {
return "rendered token does not match live"
}
// sourceFormRefusal turns a projection that could not place an edit into the same reported
// refusal every other write-boundary violation surfaces as: GitPathAccepted=False / Stalled=True,
// naming the file and the object. It is not an internal error — the folder is fine and the
// operator is fine; the EDIT had nowhere honest to land, and saying so is the whole contract.
func sourceFormRefusal(filePath string, id manifestedit.Identity, err error) error {
return &manifestanalyzer.AcceptanceRefusedError{
Issues: []manifestanalyzer.AcceptanceIssue{{
Kind: manifestanalyzer.IssueUnplaceableEdit,
Path: filePath,
// Not solvable, and deliberately so: the alternative to refusing is aligning
// two lists by position, which is measurably wrong rather than merely risky
// (see the IssueUnplaceableEdit comment). Nobody can act on it.
Solvable: false,
Message: fmt.Sprintf("%s/%s in %s: %v",
id.Kind, id.Name, filePath, err),
}},
}
}
func renderFidelityRefusal(
filePath string,
id manifestedit.Identity,
fidelity *renderFidelityRefusedError,
) error {
issues := make([]manifestanalyzer.AcceptanceIssue, 0, len(fidelity.Divergences))
for _, divergence := range fidelity.Divergences {
issues = append(issues, manifestanalyzer.AcceptanceIssue{
Kind: manifestanalyzer.IssueRenderDoesNotMatchLive,
Path: filePath,
// A live value that diverges from what the folder renders is out-of-band
// substitution, not a render artifact, so whoever owns the deployment
// pipeline can reconcile the two.
Solvable: true,
Actor: manifestanalyzer.ActorPlatformOperator,
Field: divergence.Field,
Token: divergence.Token,
Message: fmt.Sprintf("%s/%s in %s: rendered token %q at %s does not match live",
id.Kind, id.Name, filePath, divergence.Token, divergence.Field),
})
}
return &manifestanalyzer.AcceptanceRefusedError{Issues: issues}
}
// renderPrecondition is the oracle: it runs once the whole plan is known and before a byte is
// touched, so a refusal aborts the flush and commits nothing. It only runs when the flush routed
// something through a kustomization, so a repo with no override chain pays nothing.
//
// A refusal is an AcceptanceRefusedError, surfacing as GitPathAccepted=False / Stalled=True with
// the file and object named. Silently not mirroring a resource is the failure this path exists to
// prevent, so it must not be the failure it introduces.
func (wb *writeBatch) renderPrecondition() error {
if !wb.putToKustomize {
return nil
}