Skip to content

Synchronize background process actions disable in TCMalloc. #1015

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Synchronize background process actions disable in TCMalloc. #1015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 119 additions & 95 deletions tcmalloc/background.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,30 @@
#include "tcmalloc/static_vars.h"
#include "tcmalloc/stats.h"

namespace tcmalloc {
namespace tcmalloc_internal {
namespace {

ABSL_CONST_INIT absl::Mutex background_mutex(absl::kConstInit);

bool BackgroundProcessActionsDisabled(void*) {
return !Parameters::background_process_actions_enabled();
}

} // namespace
} // namespace tcmalloc_internal
} // namespace tcmalloc

extern "C" {

void MallocExtension_Internal_SetBackgroundProcessActionsEnabled(bool value) {
absl::MutexLock l(tcmalloc::tcmalloc_internal::background_mutex);
tcmalloc::tcmalloc_internal::Parameters::
set_background_process_actions_enabled(value);
}

} // extern "C"

// Release memory to the system at a constant rate.
void MallocExtension_Internal_ProcessBackgroundActions() {
using ::tcmalloc::tcmalloc_internal::Parameters;
Expand All @@ -58,6 +82,10 @@ void MallocExtension_Internal_ProcessBackgroundActions() {
// want to separately account for pages released by ProcessBackgroundActions.
tcmalloc::tcmalloc_internal::ConstantRatePageAllocatorReleaser releaser;

// We guard actions under background_mutex to ensure concurrent invocations
// are serialized and SetBackgroundProcessActionsEnabled(false) waits for any
// in-flight iteration to complete.
absl::MutexLock l(tcmalloc::tcmalloc_internal::background_mutex);
while (tcmalloc::MallocExtension::GetBackgroundProcessActionsEnabled()) {
const absl::Duration sleep_time =
tcmalloc::MallocExtension::GetBackgroundProcessSleepInterval();
Expand Down Expand Up @@ -98,112 +126,108 @@ void MallocExtension_Internal_ProcessBackgroundActions() {

absl::Time now = absl::Now();

// TODO(b/278618299): We guard various actions under a single lock, since
// individual operations may not be amenable to concurrent operations.
{
ABSL_CONST_INIT static absl::Mutex mu(absl::kConstInit);
absl::MutexLock l(mu);

// We follow the cache hierarchy in TCMalloc from outermost (per-CPU) to
// innermost (the page heap). Freeing up objects at one layer can help
// aid memory coalescing for inner caches.

if (tcmalloc::MallocExtension::PerCpuCachesActive()) {
// Accelerate fences as part of this operation by registering this
// thread with rseq. While this is not strictly required to succeed, we
// do not expect an inconsistent state for rseq (some threads registered
// and some threads unable to).
TC_CHECK(tcmalloc::tcmalloc_internal::subtle::percpu::IsFast());

// Try to drain per-cpu caches once every cpu_cache_drain_period
// when enabled.
if (now - last_drain >= cpu_cache_drain_period) {
tc_globals.cpu_cache().TryDrainingCaches();
last_drain = now;
}

if (now - last_shuffle >= cpu_cache_shuffle_period) {
tc_globals.cpu_cache().ShuffleCpuCaches();
last_shuffle = now;
}

if (now - last_size_class_resize >= size_class_resize_period) {
tc_globals.cpu_cache().ResizeSizeClasses();
last_size_class_resize = now;
}

if (Parameters::resize_size_class_max_capacity() &&
now - last_size_class_max_capacity_resize >=
size_class_max_capacity_resize_period) {
tc_globals.cpu_cache().ResizeSizeClassMaxCapacities();
last_size_class_max_capacity_resize = now;
}

// See if we need to grow the slab once every kCpuCacheSlabResizePeriod
// when enabled.
if (Parameters::per_cpu_caches_dynamic_slab_enabled() &&
now - last_slab_resize_check >= cpu_cache_slab_resize_period) {
tc_globals.cpu_cache().ResizeSlabIfNeeded();
last_slab_resize_check = now;
}

tc_globals.cpu_cache().ClearTouchedCpus();
// We follow the cache hierarchy in TCMalloc from outermost (per-CPU) to
// innermost (the page heap). Freeing up objects at one layer can help
// aid memory coalescing for inner caches.

if (tcmalloc::MallocExtension::PerCpuCachesActive()) {
// Accelerate fences as part of this operation by registering this
// thread with rseq. While this is not strictly required to succeed, we
// do not expect an inconsistent state for rseq (some threads registered
// and some threads unable to).
TC_CHECK(tcmalloc::tcmalloc_internal::subtle::percpu::IsFast());

// Try to drain per-cpu caches once every cpu_cache_drain_period
// when enabled.
if (now - last_drain >= cpu_cache_drain_period) {
tc_globals.cpu_cache().TryDrainingCaches();
last_drain = now;
}

tc_globals.sharded_transfer_cache().Plunder();

#ifndef TCMALLOC_INTERNAL_SMALL_BUT_SLOW
// Try to plunder and reclaim unused objects from transfer caches.
if (now - last_transfer_cache_plunder_check >=
transfer_cache_plunder_period) {
tc_globals.transfer_cache().TryPlunder();
last_transfer_cache_plunder_check = now;
if (now - last_shuffle >= cpu_cache_shuffle_period) {
tc_globals.cpu_cache().ShuffleCpuCaches();
last_shuffle = now;
}

if (now - last_transfer_cache_resize_check >=
transfer_cache_resize_period) {
tc_globals.transfer_cache().TryResizingCaches();
last_transfer_cache_resize_check = now;
if (now - last_size_class_resize >= size_class_resize_period) {
tc_globals.cpu_cache().ResizeSizeClasses();
last_size_class_resize = now;
}
#endif

if (now - last_hpaa_hugepage_check >= hpaa_hugepage_check_period) {
tc_globals.page_allocator().TreatHugepageTrackers(
Parameters::usermode_hugepage_collapse());
last_hpaa_hugepage_check = now;
if (Parameters::resize_size_class_max_capacity() &&
now - last_size_class_max_capacity_resize >=
size_class_max_capacity_resize_period) {
tc_globals.cpu_cache().ResizeSizeClassMaxCapacities();
last_size_class_max_capacity_resize = now;
}

// If time goes backwards, we would like to cap the release rate at 0.
//
// TODO(b/495452446): Improve test coverage and possibly move to working
// integer space entirely.
double calculated_bytes =
static_cast<size_t>(Parameters::background_release_rate()) *
absl::ToDoubleSeconds(now - prev_time);
constexpr double kMaxSsize =
static_cast<double>(std::numeric_limits<ssize_t>::max());

ssize_t bytes_to_release = calculated_bytes >= kMaxSsize
? std::numeric_limits<ssize_t>::max()
: calculated_bytes;
bytes_to_release = std::max<ssize_t>(bytes_to_release, 0);

// If release rate is set to 0, do not release memory to system. However,
// if we want to release free and backed hugepages from HugeRegion or
// follow HugePageFiller's smoothing of demand,
// ReleaseMemoryToSystem should be able to release those pages to the
// system even with bytes_to_release = 0.
if (bytes_to_release > 0 ||
Parameters::release_pages_from_huge_region() ||
Parameters::release_max_filler_pages()) {
releaser.Release(bytes_to_release,
/*reason=*/tcmalloc::tcmalloc_internal::
PageReleaseReason::kProcessBackgroundActions);
// See if we need to grow the slab once every kCpuCacheSlabResizePeriod
// when enabled.
if (Parameters::per_cpu_caches_dynamic_slab_enabled() &&
now - last_slab_resize_check >= cpu_cache_slab_resize_period) {
tc_globals.cpu_cache().ResizeSlabIfNeeded();
last_slab_resize_check = now;
}

prev_time = now;
tc_globals.cpu_cache().ClearTouchedCpus();
}

absl::SleepFor(sleep_time);
tc_globals.sharded_transfer_cache().Plunder();

#ifndef TCMALLOC_INTERNAL_SMALL_BUT_SLOW
// Try to plunder and reclaim unused objects from transfer caches.
if (now - last_transfer_cache_plunder_check >=
transfer_cache_plunder_period) {
tc_globals.transfer_cache().TryPlunder();
last_transfer_cache_plunder_check = now;
}

if (now - last_transfer_cache_resize_check >=
transfer_cache_resize_period) {
tc_globals.transfer_cache().TryResizingCaches();
last_transfer_cache_resize_check = now;
}
#endif

if (now - last_hpaa_hugepage_check >= hpaa_hugepage_check_period) {
tc_globals.page_allocator().TreatHugepageTrackers(
Parameters::usermode_hugepage_collapse());
last_hpaa_hugepage_check = now;
}

// If time goes backwards, we would like to cap the release rate at 0.
//
// TODO(b/495452446): Improve test coverage and possibly move to working
// integer space entirely.
double calculated_bytes =
static_cast<size_t>(Parameters::background_release_rate()) *
absl::ToDoubleSeconds(now - prev_time);
constexpr double kMaxSsize =
static_cast<double>(std::numeric_limits<ssize_t>::max());

ssize_t bytes_to_release = calculated_bytes >= kMaxSsize
? std::numeric_limits<ssize_t>::max()
: calculated_bytes;
bytes_to_release = std::max<ssize_t>(bytes_to_release, 0);

// If release rate is set to 0, do not release memory to system. However,
// if we want to release free and backed hugepages from HugeRegion or
// follow HugePageFiller's smoothing of demand,
// ReleaseMemoryToSystem should be able to release those pages to the
// system even with bytes_to_release = 0.
if (bytes_to_release > 0 || Parameters::release_pages_from_huge_region() ||
Parameters::release_max_filler_pages()) {
releaser.Release(bytes_to_release,
/*reason=*/tcmalloc::tcmalloc_internal::
PageReleaseReason::kProcessBackgroundActions);
}

prev_time = now;

tcmalloc::tcmalloc_internal::background_mutex.AwaitWithTimeout(
absl::Condition(
&tcmalloc::tcmalloc_internal::BackgroundProcessActionsDisabled,
nullptr),
sleep_time);
}
}
10 changes: 5 additions & 5 deletions tcmalloc/parameters.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,11 @@ bool Parameters::background_process_actions_enabled() {
std::memory_order_relaxed);
}

void Parameters::set_background_process_actions_enabled(bool value) {
background_process_actions_enabled_ptr().store(value,
std::memory_order_relaxed);
}

absl::Duration Parameters::background_process_sleep_interval() {
return absl::Nanoseconds(
background_process_sleep_interval_ns().load(std::memory_order_relaxed));
Expand Down Expand Up @@ -434,11 +439,6 @@ bool MallocExtension_Internal_GetBackgroundProcessActionsEnabled() {
return Parameters::background_process_actions_enabled();
}

void MallocExtension_Internal_SetBackgroundProcessActionsEnabled(bool value) {
tcmalloc::tcmalloc_internal::background_process_actions_enabled_ptr().store(
value, std::memory_order_relaxed);
}

void MallocExtension_Internal_GetBackgroundProcessSleepInterval(
absl::Duration* ret) {
*ret = Parameters::background_process_sleep_interval();
Expand Down
1 change: 1 addition & 0 deletions tcmalloc/parameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ class Parameters {
}

static bool background_process_actions_enabled();
static void set_background_process_actions_enabled(bool value);
static absl::Duration background_process_sleep_interval();

static void set_filler_skip_subrelease_short_interval(absl::Duration value) {
Expand Down
23 changes: 23 additions & 0 deletions tcmalloc/testing/background_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,29 @@ TEST(BackgroundTest, Stress) {
background.join();
}

TEST(BackgroundTest, DisableInterruptsSleep) {
// Use a long sleep interval so that without early wakeup, the thread would
// block for a significant duration.
ScopedBackgroundProcessSleepInterval sleep_time(absl::Hours(1));

std::thread background([]() { MallocExtension::ProcessBackgroundActions(); });

// Give the background thread time to start and enter sleep.
absl::SleepFor(absl::Milliseconds(100));

const absl::Time start = absl::Now();
{
ScopedBackgroundProcessActionsEnabled background_process_enabled(
/*value=*/false);
background.join();
}
const absl::Duration elapsed = absl::Now() - start;

// The background thread should wake up and exit promptly rather than waiting
// for the 1-hour sleep interval.
EXPECT_LT(elapsed, absl::Seconds(10));
}

} // namespace
} // namespace tcmalloc

Expand Down
6 changes: 0 additions & 6 deletions tcmalloc/testing/testutil.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,6 @@ class ScopedBackgroundProcessActionsEnabled {
explicit ScopedBackgroundProcessActionsEnabled(bool value)
: previous_(MallocExtension::GetBackgroundProcessActionsEnabled()) {
MallocExtension::SetBackgroundProcessActionsEnabled(value);
if (!value) {
// If the thread is doing something at the moment, let it finish.
// This is unreliable, but simple. The thread sleeps for 1 sec,
// so we sleep for 5.
absl::SleepFor(absl::Seconds(5));
}
}

~ScopedBackgroundProcessActionsEnabled() {
Expand Down
Loading