Add size-gated hugepage optimization for large allocations

tcmalloc/internal/parameter_accessors.h

@@ -109,6 +109,8 @@ ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetUseUserspaceCollapseHeuristics(
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHeapPartitioning();
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetBackSmallAllocations();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSmallAllocations(bool v);
+ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled();
+ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v);
 ABSL_ATTRIBUTE_WEAK int32_t TCMalloc_Internal_GetBackSizeThresholdBytes();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v);
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetEnableUnfilteredCollapse();

tcmalloc/internal/system_allocator.h

@@ -377,6 +377,13 @@ SystemAllocator<Topology, NormalPartitions>::MmapRegion::Alloc(
     // This is only advisory, so ignore the error.
     ErrnoRestorer errno_restorer;
     (void)madvise(result_ptr, actual_size, MADV_NOHUGEPAGE);
+  } else if (Parameters::madvise_hugepage_mmap_enabled()) {
+    // Opt-in to transparent hugepages when system is 
+    // configured for transparent_hugepage=madvise. This can improve memory
+    // performance by reducing TLB pressure.
+    // This is only advisory, so ignore the error.
+    ErrnoRestorer errno_restorer;
+    (void)madvise(result_ptr, actual_size, MADV_HUGEPAGE);
   }
   free_size_ -= actual_size;
   return {result_ptr, actual_size};

tcmalloc/parameters.cc

@@ -220,6 +220,8 @@ ABSL_CONST_INIT std::atomic<int32_t> Parameters::back_size_threshold_bytes_(
     kPageSize);
 ABSL_CONST_INIT std::atomic<bool> Parameters::enable_unfiltered_collapse_(
     false);
+ABSL_CONST_INIT std::atomic<bool> Parameters::madvise_hugepage_mmap_enabled_(
+    false);
 
 static std::atomic<bool>& heap_partitioning_enabled() {
   ABSL_CONST_INIT static absl::once_flag flag;
@@ -318,6 +320,10 @@ bool TCMalloc_Internal_GetBackSmallAllocations() {
   return Parameters::back_small_allocations();
 }
 
+bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled() {
+  return Parameters::madvise_hugepage_mmap_enabled();
+}
+
 int ABSL_ATTRIBUTE_WEAK default_want_disable_dynamic_slabs();
 
 // TODO(b/271475288): remove the default_want_disable_dynamic_slabs opt-out
@@ -620,6 +626,10 @@ void TCMalloc_Internal_SetBackSmallAllocations(bool v) {
   Parameters::back_small_allocations_.store(v, std::memory_order_relaxed);
 }
 
+void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v) {
+  Parameters::madvise_hugepage_mmap_enabled_.store(v, std::memory_order_relaxed);
+}
+
 void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v) {
   Parameters::back_size_threshold_bytes_.store(v, std::memory_order_relaxed);
 }

tcmalloc/parameters.h

@@ -191,6 +191,10 @@ class Parameters {
     TCMalloc_Internal_SetPerCpuCachesDynamicSlabShrinkThreshold(value);
   }
 
+  static bool madvise_hugepage_mmap_enabled() {
+    return madvise_hugepage_mmap_enabled_.load(std::memory_order_relaxed);
+  }
+
   static bool heap_partitioning();
 
   static central_freelist_internal::PriorityListLength priority_list_length();
@@ -231,6 +235,7 @@ class Parameters {
   friend void ::TCMalloc_Internal_SetBackSmallAllocations(bool v);
   friend void ::TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v);
   friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v);
+  friend void ::TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v);
 
   static std::atomic<MallocExtension::BytesPerSecond> background_release_rate_;
   static std::atomic<int64_t> guarded_sampling_interval_;
@@ -252,6 +257,7 @@ class Parameters {
   static std::atomic<bool> back_small_allocations_;
   static std::atomic<int32_t> back_size_threshold_bytes_;
   static std::atomic<bool> enable_unfiltered_collapse_;
+  static std::atomic<bool> madvise_hugepage_mmap_enabled_;
 };
 
 }  // namespace tcmalloc_internal