From 9576476b23e81aaea58099d58362c21e59e8761b Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 1 Aug 2026 13:33:59 +0200 Subject: [PATCH 1/5] Reduce false sharing in MemoryCache and CounterAggregator Follow-up to #131470. MemoryCache: CoherentState._cacheSize is Interlocked-updated on every Set/Remove when SizeLimit is configured, while _stringEntries and _nonStringEntries are read on every TryGetValue. Move the counter onto its own cache line so the write-hot atomic stops invalidating the line holding those read-mostly references. CounterAggregator: move PaddedDouble.Value to the end of its padding. At offset 0, element 0's CAS target shared a cache line with the array's Length field, which the bounds check in Update loads on every call from every thread - the same hazard #131470 fixed in Counters. The struct stays 64 bytes, so this costs no extra memory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3cea9f94-d161-471d-9fd3-543f6148300d --- .../src/MemoryCache.cs | 36 +++++++++++++------ ...Microsoft.Extensions.Caching.Memory.csproj | 4 +++ .../Diagnostics/Metrics/CounterAggregator.cs | 23 +++++++++--- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs b/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs index 51b5c3fe523e64..3b85144acab4ef 100644 --- a/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs +++ b/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs @@ -37,6 +37,8 @@ public class MemoryCache : IMemoryCache private bool _disposed; private DateTime _lastExpirationScan; + private const int CacheLineSize = global::Internal.PaddingHelpers.CACHE_LINE_SIZE; + /// /// Creates a new instance. /// @@ -186,8 +188,8 @@ internal void SetEntry(CacheEntry entry) // exactly once, tied to the swap we performed. Doing this speculatively // inside UpdateCacheSizeExceedsCapacity (before the swap) races with a // concurrent RemoveEntry of the prior entry and double-counts the decrement, - // drifting _cacheSize negative and permanently blocking all future inserts. - Interlocked.Add(ref coherentState._cacheSize, -priorEntry.Size); + // drifting CacheSize negative and permanently blocking all future inserts. + Interlocked.Add(ref coherentState.CacheSize, -priorEntry.Size); } } else @@ -209,7 +211,7 @@ internal void SetEntry(CacheEntry entry) if (_options.HasSizeLimit) { // Entry could not be added, roll back the size increment for this entry only. - Interlocked.Add(ref coherentState._cacheSize, -entry.Size); + Interlocked.Add(ref coherentState.CacheSize, -entry.Size); } entry.SetExpired(EvictionReason.Replaced); entry.InvokeEvictionCallbacks(); @@ -361,7 +363,7 @@ public void Remove(object key) { if (_options.HasSizeLimit) { - Interlocked.Add(ref coherentState._cacheSize, -entry.Size); + Interlocked.Add(ref coherentState.CacheSize, -entry.Size); } entry.SetExpired(EvictionReason.Removed); @@ -554,11 +556,11 @@ private bool UpdateCacheSizeExceedsCapacity(CacheEntry entry, CacheEntry? priorE { // The capacity decision still accounts for the prior entry being replaced (its size is // freed by the replace), so a same-or-smaller replacement at the size limit is admitted. - // However, only the new entry's size is committed to _cacheSize here. The prior entry's + // However, only the new entry's size is committed to CacheSize here. The prior entry's // size is decremented by the caller, atomically with the dictionary swap that actually // removes it. Decrementing the prior size here (before the swap) races with a concurrent // RemoveEntry of the same prior entry and double-counts the decrement, drifting - // _cacheSize negative and permanently blocking all future inserts. + // CacheSize negative and permanently blocking all future inserts. long sizeAfterReplace = sizeRead + entry.Size - priorSize; if ((ulong)sizeAfterReplace > (ulong)sizeLimit) @@ -568,7 +570,7 @@ private bool UpdateCacheSizeExceedsCapacity(CacheEntry entry, CacheEntry? priorE } long committedSize = sizeRead + entry.Size; - long original = Interlocked.CompareExchange(ref coherentState._cacheSize, committedSize, sizeRead); + long original = Interlocked.CompareExchange(ref coherentState.CacheSize, committedSize, sizeRead); if (sizeRead == original) { return false; @@ -798,7 +800,21 @@ public CoherentState() } #endif - internal long _cacheSize; + // CacheSize is Interlocked-updated on every Set/Remove when SizeLimit is set, while + // _stringEntries/_nonStringEntries are read on every TryGetValue; the padding keeps the + // write-hot atomic off the line holding those read-mostly references. It has to live in + // a struct -- layout attributes on a class only affect marshaling, and the runtime + // reorders class fields freely -- with Value one line in and another line after it, so + // no neighbouring field can share its line whatever offset the struct lands on. + private CacheSizePadded _cacheSizePadded; + + [StructLayout(LayoutKind.Explicit, Size = CacheLineSize * 2)] + private struct CacheSizePadded + { + [FieldOffset(CacheLineSize)] public long Value; + } + + internal ref long CacheSize => ref _cacheSizePadded.Value; internal bool TryGetValue(object key, [NotNullWhen(true)] out CacheEntry? entry) => key is string s ? _stringEntries.TryGetValue(s, out entry) : _nonStringEntries.TryGetValue(key, out entry); @@ -848,7 +864,7 @@ public IEnumerable GetAllKeys() internal int Count => _stringEntries.Count + _nonStringEntries.Count; - internal long Size => Volatile.Read(ref _cacheSize); + internal long Size => Volatile.Read(ref CacheSize); internal bool RemoveEntry(CacheEntry entry, MemoryCacheOptions options) { @@ -862,7 +878,7 @@ internal bool RemoveEntry(CacheEntry entry, MemoryCacheOptions options) { if (options.HasSizeLimit) { - Interlocked.Add(ref _cacheSize, -entry.Size); + Interlocked.Add(ref CacheSize, -entry.Size); } entry.InvokeEvictionCallbacks(); return true; diff --git a/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj b/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj index f9e5dfb910cfa4..506e4935142320 100644 --- a/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj +++ b/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj @@ -24,6 +24,10 @@ + + + + diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs index 4577c16b9b988e..25b8f8dabf0570 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs @@ -72,13 +72,28 @@ public override IAggregationStatistics Collect() return new CounterStatistics(delta, _isMonotonic, aggregatedValue); } - // 64 bytes is the size of a cache line on many systems. We pad the double to false sharing. - // For the rare systems with a larger cache line, we may simply incur a little more false - // sharing. This is a trade-off between throughput and memory footprint. + // Each delta gets its own 64-byte cache line so that cores assigned to adjacent slots do + // not contend. Value sits at the *end* of the padding rather than the start: the array's + // Length field lives at offset 8 of the array object and the first element starts at + // offset 16, so a Value at offset 0 would put element 0 in the same line as Length. The + // bounds check in Update performs a plain load of Length on every call from every thread, + // and having that load target the same line as a contended atomic RMW is what defeats + // far-atomic handling on Arm64. At offset 56 the gap from Length to Value[0] is exactly 64, + // so they always land in different lines. + // + // The trade is that the last element's Value ends flush with the array object, so it shares + // a line with whatever the GC places next. That is the better end to give up: it costs one + // slot only when the neighbouring object happens to be write-hot, whereas the Length + // collision it replaces was hit by every Update on every thread. + // + // 64 is hardcoded rather than taken from PaddingHelpers.CACHE_LINE_SIZE because that + // constant is 128 for out-of-CoreLib libraries, which would double this array for no + // benefit on the 64-byte-line hardware this runs on. As before, systems with larger lines + // may incur a little more false sharing between adjacent slots. [StructLayout(LayoutKind.Explicit, Size = 64)] private struct PaddedDouble { - [FieldOffset(0)] + [FieldOffset(64 - sizeof(double))] public double Value; } } From 2d287e1c202dda0a65b82ada9de07e7e4d6d97fc Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 1 Aug 2026 13:41:30 +0200 Subject: [PATCH 2/5] Trim the PaddedDouble comment Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3cea9f94-d161-471d-9fd3-543f6148300d --- .../Diagnostics/Metrics/CounterAggregator.cs | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs index 25b8f8dabf0570..89852f38d18075 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs @@ -72,24 +72,11 @@ public override IAggregationStatistics Collect() return new CounterStatistics(delta, _isMonotonic, aggregatedValue); } - // Each delta gets its own 64-byte cache line so that cores assigned to adjacent slots do - // not contend. Value sits at the *end* of the padding rather than the start: the array's - // Length field lives at offset 8 of the array object and the first element starts at - // offset 16, so a Value at offset 0 would put element 0 in the same line as Length. The - // bounds check in Update performs a plain load of Length on every call from every thread, - // and having that load target the same line as a contended atomic RMW is what defeats - // far-atomic handling on Arm64. At offset 56 the gap from Length to Value[0] is exactly 64, - // so they always land in different lines. - // - // The trade is that the last element's Value ends flush with the array object, so it shares - // a line with whatever the GC places next. That is the better end to give up: it costs one - // slot only when the neighbouring object happens to be write-hot, whereas the Length - // collision it replaces was hit by every Update on every thread. - // - // 64 is hardcoded rather than taken from PaddingHelpers.CACHE_LINE_SIZE because that - // constant is 128 for out-of-CoreLib libraries, which would double this array for no - // benefit on the 64-byte-line hardware this runs on. As before, systems with larger lines - // may incur a little more false sharing between adjacent slots. + // 64 bytes is the size of a cache line on many systems; larger ones may see a little more + // false sharing. Value sits at the end rather than at offset 0 so element 0 doesn't share a + // line with the array's Length field at offset 8: the bounds check in Update loads Length on + // every call from every thread, and a plain load of a line that is also a contended atomic's + // target defeats far-atomic handling on Arm64. [StructLayout(LayoutKind.Explicit, Size = 64)] private struct PaddedDouble { From a718a31e4e41cb270bacd759c38cff2117ddface Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 1 Aug 2026 13:45:48 +0200 Subject: [PATCH 3/5] Hardcode 64 for CacheSizePadded CACHE_LINE_SIZE is 128 unconditionally for out-of-CoreLib libraries, which made the struct 256 bytes per MemoryCache on every platform. Match CounterAggregator and use 64 directly, which also drops the Padding.cs dependency. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3cea9f94-d161-471d-9fd3-543f6148300d --- .../src/MemoryCache.cs | 10 ++++------ .../src/Microsoft.Extensions.Caching.Memory.csproj | 4 ---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs b/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs index 3b85144acab4ef..2c21f411b32fdd 100644 --- a/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs +++ b/src/libraries/Microsoft.Extensions.Caching.Memory/src/MemoryCache.cs @@ -37,8 +37,6 @@ public class MemoryCache : IMemoryCache private bool _disposed; private DateTime _lastExpirationScan; - private const int CacheLineSize = global::Internal.PaddingHelpers.CACHE_LINE_SIZE; - /// /// Creates a new instance. /// @@ -804,14 +802,14 @@ public CoherentState() // _stringEntries/_nonStringEntries are read on every TryGetValue; the padding keeps the // write-hot atomic off the line holding those read-mostly references. It has to live in // a struct -- layout attributes on a class only affect marshaling, and the runtime - // reorders class fields freely -- with Value one line in and another line after it, so - // no neighbouring field can share its line whatever offset the struct lands on. + // reorders class fields freely. 64 is the size of a cache line on many systems; larger + // ones may see a little more false sharing. private CacheSizePadded _cacheSizePadded; - [StructLayout(LayoutKind.Explicit, Size = CacheLineSize * 2)] + [StructLayout(LayoutKind.Explicit, Size = 128)] private struct CacheSizePadded { - [FieldOffset(CacheLineSize)] public long Value; + [FieldOffset(64)] public long Value; } internal ref long CacheSize => ref _cacheSizePadded.Value; diff --git a/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj b/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj index 506e4935142320..f9e5dfb910cfa4 100644 --- a/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj +++ b/src/libraries/Microsoft.Extensions.Caching.Memory/src/Microsoft.Extensions.Caching.Memory.csproj @@ -24,10 +24,6 @@ - - - - From a977a4f3092ce870cd4743a6df5b8eb06bc28fb5 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 1 Aug 2026 13:48:38 +0200 Subject: [PATCH 4/5] Don't hardcode array offsets in the PaddedDouble comment The specific offsets are 64-bit CoreCLR layout; the reason Value sits at the end holds regardless. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3cea9f94-d161-471d-9fd3-543f6148300d --- .../src/System/Diagnostics/Metrics/CounterAggregator.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs index 89852f38d18075..78e0a4a14cfdc9 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs @@ -74,9 +74,9 @@ public override IAggregationStatistics Collect() // 64 bytes is the size of a cache line on many systems; larger ones may see a little more // false sharing. Value sits at the end rather than at offset 0 so element 0 doesn't share a - // line with the array's Length field at offset 8: the bounds check in Update loads Length on - // every call from every thread, and a plain load of a line that is also a contended atomic's - // target defeats far-atomic handling on Arm64. + // line with the array's Length field, which the bounds check in Update loads on every call + // from every thread -- a plain load of a line that is also a contended atomic's target + // defeats far-atomic handling on Arm64. [StructLayout(LayoutKind.Explicit, Size = 64)] private struct PaddedDouble { From 6850f1f8dbff1b9e71c8e895bf7749c3b482f519 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 1 Aug 2026 14:54:11 +0200 Subject: [PATCH 5/5] Drop the CounterAggregator change Benchmarks show it trades one false-sharing hazard for another rather than removing it. Moving Value to the end of the 64-byte element leaves the last slot flush against the array object's end (16 + 7*64 + 56 + 8 = 528 = object size), so it shares a line with whatever the GC places next. Which hazard bites depends on heap layout: 3.12x faster on linux-arm64 Cobalt 100, 3.75x slower on windows-arm64 on the same CPU. Keeping only the MemoryCache change, which is a consistent win on all three targets with a flat control arm. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3cea9f94-d161-471d-9fd3-543f6148300d --- .../System/Diagnostics/Metrics/CounterAggregator.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs index 78e0a4a14cfdc9..4577c16b9b988e 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/CounterAggregator.cs @@ -72,15 +72,13 @@ public override IAggregationStatistics Collect() return new CounterStatistics(delta, _isMonotonic, aggregatedValue); } - // 64 bytes is the size of a cache line on many systems; larger ones may see a little more - // false sharing. Value sits at the end rather than at offset 0 so element 0 doesn't share a - // line with the array's Length field, which the bounds check in Update loads on every call - // from every thread -- a plain load of a line that is also a contended atomic's target - // defeats far-atomic handling on Arm64. + // 64 bytes is the size of a cache line on many systems. We pad the double to false sharing. + // For the rare systems with a larger cache line, we may simply incur a little more false + // sharing. This is a trade-off between throughput and memory footprint. [StructLayout(LayoutKind.Explicit, Size = 64)] private struct PaddedDouble { - [FieldOffset(64 - sizeof(double))] + [FieldOffset(0)] public double Value; } }