diff --git a/tcmalloc/internal/parameter_accessors.h b/tcmalloc/internal/parameter_accessors.h index 871fde6a6..9faced8df 100644 --- a/tcmalloc/internal/parameter_accessors.h +++ b/tcmalloc/internal/parameter_accessors.h @@ -109,6 +109,8 @@ ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetUseUserspaceCollapseHeuristics( ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHeapPartitioning(); ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetBackSmallAllocations(); ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSmallAllocations(bool v); +ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled(); +ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v); ABSL_ATTRIBUTE_WEAK int32_t TCMalloc_Internal_GetBackSizeThresholdBytes(); ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v); ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetEnableUnfilteredCollapse(); diff --git a/tcmalloc/internal/system_allocator.h b/tcmalloc/internal/system_allocator.h index a25e1f1b4..fd2cf72cc 100644 --- a/tcmalloc/internal/system_allocator.h +++ b/tcmalloc/internal/system_allocator.h @@ -377,6 +377,13 @@ SystemAllocator::MmapRegion::Alloc( // This is only advisory, so ignore the error. ErrnoRestorer errno_restorer; (void)madvise(result_ptr, actual_size, MADV_NOHUGEPAGE); + } else if (Parameters::madvise_hugepage_mmap_enabled()) { + // Opt-in to transparent hugepages when system is + // configured for transparent_hugepage=madvise. This can improve memory + // performance by reducing TLB pressure. + // This is only advisory, so ignore the error. + ErrnoRestorer errno_restorer; + (void)madvise(result_ptr, actual_size, MADV_HUGEPAGE); } free_size_ -= actual_size; return {result_ptr, actual_size}; diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 005897d4a..fce5a6bd4 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -220,6 +220,8 @@ ABSL_CONST_INIT std::atomic Parameters::back_size_threshold_bytes_( kPageSize); ABSL_CONST_INIT std::atomic Parameters::enable_unfiltered_collapse_( false); +ABSL_CONST_INIT std::atomic Parameters::madvise_hugepage_mmap_enabled_( + false); static std::atomic& heap_partitioning_enabled() { ABSL_CONST_INIT static absl::once_flag flag; @@ -318,6 +320,10 @@ bool TCMalloc_Internal_GetBackSmallAllocations() { return Parameters::back_small_allocations(); } +bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled() { + return Parameters::madvise_hugepage_mmap_enabled(); +} + int ABSL_ATTRIBUTE_WEAK default_want_disable_dynamic_slabs(); // TODO(b/271475288): remove the default_want_disable_dynamic_slabs opt-out @@ -620,6 +626,10 @@ void TCMalloc_Internal_SetBackSmallAllocations(bool v) { Parameters::back_small_allocations_.store(v, std::memory_order_relaxed); } +void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v) { + Parameters::madvise_hugepage_mmap_enabled_.store(v, std::memory_order_relaxed); +} + void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v) { Parameters::back_size_threshold_bytes_.store(v, std::memory_order_relaxed); } diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 5af17d615..f3f62be7a 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -191,6 +191,10 @@ class Parameters { TCMalloc_Internal_SetPerCpuCachesDynamicSlabShrinkThreshold(value); } + static bool madvise_hugepage_mmap_enabled() { + return madvise_hugepage_mmap_enabled_.load(std::memory_order_relaxed); + } + static bool heap_partitioning(); static central_freelist_internal::PriorityListLength priority_list_length(); @@ -231,6 +235,7 @@ class Parameters { friend void ::TCMalloc_Internal_SetBackSmallAllocations(bool v); friend void ::TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v); friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v); + friend void ::TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v); static std::atomic background_release_rate_; static std::atomic guarded_sampling_interval_; @@ -252,6 +257,7 @@ class Parameters { static std::atomic back_small_allocations_; static std::atomic back_size_threshold_bytes_; static std::atomic enable_unfiltered_collapse_; + static std::atomic madvise_hugepage_mmap_enabled_; }; } // namespace tcmalloc_internal