From ebd7553ea22254d9ef5ca6c11e97021b837b84e8 Mon Sep 17 00:00:00 2001 From: Chris Kennelly CA Date: Mon, 21 Sep 2026 13:18:06 -0700 Subject: [PATCH] Eliminate harness measurement distortion in TransferCache benchmarks. BM_InsertRange, BM_RemoveRange, BM_RealisticBatchNonBatchMutations, and BM_RealisticHitRate in transfer_cache_benchmark invoked state.PauseTiming() and state.ResumeTiming() inside every iteration of the benchmark loop, spending ~60% of CPU cycles in clock_gettime/vDSO syscall overhead (~1300 ns/iter) and obscuring the underlying 10-30 ns cache operations. In addition, FakeTransferCacheEnvironment::Insert and Remove allocated temporary std::vector instances on the heap in each call. Amortize setup across batches with state.KeepRunningBatch in BM_InsertRange and BM_RemoveRange, precompute PRNG decisions in BM_RealisticBatchNonBatchMutations and BM_RealisticHitRate using compile-time constant modulo, and replace dynamic std::vector buffers with stack arrays in FakeTransferCacheEnvironment. PiperOrigin-RevId: 985469423 --- tcmalloc/mock_transfer_cache.h | 22 +++-- tcmalloc/transfer_cache_benchmark.cc | 116 ++++++++++++++++----------- 2 files changed, 77 insertions(+), 61 deletions(-) diff --git a/tcmalloc/mock_transfer_cache.h b/tcmalloc/mock_transfer_cache.h index d71783f0a..dc3beb291 100644 --- a/tcmalloc/mock_transfer_cache.h +++ b/tcmalloc/mock_transfer_cache.h @@ -153,26 +153,24 @@ class FakeTransferCacheEnvironment { bool Grow() { return cache_.IncreaseCacheCapacity(kSizeClass); } void Insert(int n, int batch = kBatchSize) { - std::vector bufs; + void* bufs[kMaxObjectsToMove]; while (n > 0) { - int b = std::min(n, batch); - bufs.resize(b); - central_freelist().AllocateBatch(absl::MakeSpan(bufs)); - cache_.InsertRange(kSizeClass, absl::MakeSpan(bufs)); + int b = std::min({n, batch, static_cast(kMaxObjectsToMove)}); + central_freelist().AllocateBatch(absl::MakeSpan(bufs, b)); + cache_.InsertRange(kSizeClass, absl::MakeSpan(bufs, b)); n -= b; } } void Remove(int n, int batch = kBatchSize) { - std::vector bufs; + void* bufs[kMaxObjectsToMove]; while (n > 0) { - int b = std::min(n, batch); - bufs.resize(b); - int removed = cache_.RemoveRange(kSizeClass, absl::MakeSpan(bufs)); + int b = std::min({n, batch, static_cast(kMaxObjectsToMove)}); + int removed = cache_.RemoveRange(kSizeClass, absl::MakeSpan(bufs, b)); // Ensure we make progress. - ASSERT_GT(removed, 0); - ASSERT_LE(removed, b); - central_freelist().FreeBatch({&bufs[0], static_cast(removed)}); + TC_ASSERT_GT(removed, 0); + TC_ASSERT_LE(removed, b); + central_freelist().FreeBatch({bufs, static_cast(removed)}); n -= removed; } } diff --git a/tcmalloc/transfer_cache_benchmark.cc b/tcmalloc/transfer_cache_benchmark.cc index fbe2dede2..d3f8f6c54 100644 --- a/tcmalloc/transfer_cache_benchmark.cc +++ b/tcmalloc/transfer_cache_benchmark.cc @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include +#include #include "absl/random/distributions.h" #include "absl/random/random.h" @@ -98,20 +100,26 @@ template void BM_InsertRange(benchmark::State& state) { const int kBatchSize = Env::kBatchSize; const int kMaxObjectsToMove = Env::kMaxObjectsToMove; + constexpr int kBatches = 16; - // optional to have more precise control of when the destruction occurs, as - // we want to avoid polluting the timing with the dtor. - std::optional e; - void* batch[kMaxObjectsToMove]; - for (auto iter : state) { + Env e; + void* batches[kBatches][kMaxObjectsToMove]; + for (int i = 0; i < kBatches; ++i) { + e.central_freelist().AllocateBatch( + {batches[i], static_cast(kBatchSize)}); + } + + while (state.KeepRunningBatch(kBatches)) { + for (int i = 0; i < kBatches; ++i) { + e.transfer_cache().InsertRange( + kSizeClass, {batches[i], static_cast(kBatchSize)}); + } state.PauseTiming(); - e.emplace(); - e->central_freelist().AllocateBatch({batch, kBatchSize}); - benchmark::DoNotOptimize(e); - benchmark::DoNotOptimize(batch); + for (int i = 0; i < kBatches; ++i) { + (void)e.transfer_cache().RemoveRange( + kSizeClass, {batches[i], static_cast(kBatchSize)}); + } state.ResumeTiming(); - - e->transfer_cache().InsertRange(kSizeClass, {batch, kBatchSize}); } } @@ -119,20 +127,27 @@ template void BM_RemoveRange(benchmark::State& state) { const int kBatchSize = Env::kBatchSize; const int kMaxObjectsToMove = Env::kMaxObjectsToMove; + constexpr int kBatches = 16; - // optional to have more precise control of when the destruction occurs, as - // we want to avoid polluting the timing with the dtor. - std::optional e; - void* batch[kMaxObjectsToMove]; - for (auto iter : state) { + Env e; + void* batches[kBatches][kMaxObjectsToMove]; + for (int i = 0; i < kBatches; ++i) { + e.central_freelist().AllocateBatch( + {batches[i], static_cast(kBatchSize)}); + } + + while (state.KeepRunningBatch(kBatches)) { state.PauseTiming(); - e.emplace(); - e->Insert(kBatchSize); - benchmark::DoNotOptimize(e); + for (int i = 0; i < kBatches; ++i) { + e.transfer_cache().InsertRange( + kSizeClass, {batches[i], static_cast(kBatchSize)}); + } state.ResumeTiming(); - - (void)e->transfer_cache().RemoveRange(kSizeClass, {batch, kBatchSize}); - benchmark::DoNotOptimize(batch); + for (int i = 0; i < kBatches; ++i) { + (void)e.transfer_cache().RemoveRange( + kSizeClass, {batches[i], static_cast(kBatchSize)}); + benchmark::DoNotOptimize(batches[i]); + } } } @@ -142,11 +157,15 @@ void BM_RealisticBatchNonBatchMutations(benchmark::State& state) { Env e; absl::BitGen gen; + constexpr size_t kNumChoices = 4096; + std::array choices; + for (double& choice : choices) { + choice = absl::Uniform(gen, 0.0, 1.0); + } + size_t idx = 0; for (auto iter : state) { - state.PauseTiming(); - const double choice = absl::Uniform(gen, 0.0, 1.0); - state.ResumeTiming(); + const double choice = choices[idx++ % kNumChoices]; // These numbers have been determined by looking at production data. if (choice < 0.424) { @@ -182,34 +201,33 @@ void BM_RealisticHitRate(benchmark::State& state) { // resulting insert and remove miss rate matches that of the production. constexpr int kInterval = 5000; constexpr double kBias = 0.85; - bool insert_heavy = true; - unsigned int iterations = 0; - for (auto iter : state) { - state.PauseTiming(); - const double partial = absl::Uniform(gen, 0.0, 1.0); - // We perform insert (or remove) operations with a probability specified by - // kBias during the insert-heavy (or remove-heavy) phase of this benchmark. - const bool insert = absl::Bernoulli(gen, kBias) == insert_heavy; - state.ResumeTiming(); - if (insert) { - // These numbers have been determined by looking at production data. - if (partial < 0.65) { - e.Insert(kBatchSize); - } else { - e.Insert(1); - } - } else { - // These numbers have been determined by looking at production data. - if (partial < 0.99) { - e.Remove(kBatchSize); + struct Op { + bool insert; + int count; + }; + constexpr size_t kNumOps = 2 * kInterval; + std::vector ops; + ops.reserve(kNumOps); + for (bool insert_heavy : {true, false}) { + for (int i = 0; i < kInterval; ++i) { + const double partial = absl::Uniform(gen, 0.0, 1.0); + const bool insert = absl::Bernoulli(gen, kBias) == insert_heavy; + if (insert) { + ops.push_back({true, partial < 0.65 ? kBatchSize : 1}); } else { - e.Remove(1); + ops.push_back({false, partial < 0.99 ? kBatchSize : 1}); } } - ++iterations; - if (iterations % kInterval == 0) { - insert_heavy = !insert_heavy; + } + + size_t idx = 0; + for (auto iter : state) { + const Op& op = ops[idx++ % kNumOps]; + if (op.insert) { + e.Insert(op.count); + } else { + e.Remove(op.count); } }