Support for approx free percentage in each allocation class based on upstream AC stats implementation

(fixes getACStats call to initialize extra members of the ACStats struct).

Author: byrnedj

cachelib/allocator/CacheAllocator-inl.h

@@ -2684,6 +2684,11 @@ PoolStats CacheAllocator<CacheTrait>::getPoolStats(PoolId poolId) const {
   return ret;
 }
 
+template <typename CacheTrait>
+double CacheAllocator<CacheTrait>::slabsApproxFreePercentage(TierId tid) const {
+  return allocator_[tid]->approxFreeSlabsPercentage();
+}
+
 template <typename CacheTrait>
 ACStats CacheAllocator<CacheTrait>::getACStats(TierId tid,
                                                PoolId poolId,
@@ -2693,6 +2698,27 @@ ACStats CacheAllocator<CacheTrait>::getACStats(TierId tid,
 
   auto stats = ac.getStats();
   stats.allocLatencyNs = (*stats_.classAllocLatency)[tid][poolId][classId];
+  stats.memorySize = ac.getNumSlabs() * Slab::kSize;
+
+  if (slabsApproxFreePercentage(tid) > 0.0) {
+    auto totalMemory = MemoryAllocator::getMemorySize(memoryTierSize(tid));
+    auto freeMemory = static_cast<double>(totalMemory) * slabsApproxFreePercentage(tid) / 100.0;
+
+    // amount of free memory which has the same ratio to entire free memory as
+    // this allocation class memory size has to used memory
+    auto scaledFreeMemory = static_cast<size_t>(freeMemory * stats.memorySize / totalMemory);
+
+    auto acAllocatedMemory = (100.0 - ac.approxFreePercentage()) / 100.0 * ac.getNumSlabs() * Slab::kSize;
+    auto acMaxAvailableMemory = ac.getNumSlabs() * Slab::kSize + scaledFreeMemory;
+
+    if (acMaxAvailableMemory == 0) {
+      stats.approxFreePercent = 100.0;
+    } else {
+      stats.approxFreePercent = 100.0 - 100.0 * acAllocatedMemory / acMaxAvailableMemory;
+    }
+  } else {
+    stats.approxFreePercent = ac.approxFreePercentage();
+  }
   return stats;
 }
 
@@ -3750,6 +3776,10 @@ CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
   size_t configuredCompactCacheSize = std::accumulate(
       ccCachePoolIds.begin(), ccCachePoolIds.end(), 0ULL, addSize);
 
+  std::vector<double> slabsApproxFreePercentages;
+  for (TierId tid = 0; tid < getNumTiers(); ++tid)
+    slabsApproxFreePercentages.push_back(slabsApproxFreePercentage(tid));
+
   return CacheMemoryStats{totalCacheSize,
                           configuredTotalCacheSize,
                           configuredRegularCacheSize,
@@ -3759,7 +3789,8 @@ CacheMemoryStats CacheAllocator<CacheTrait>::getCacheMemoryStats() const {
                           allocator_[currentTier()]->getUnreservedMemorySize(),
                           nvmCache_ ? nvmCache_->getSize() : 0,
                           util::getMemAvailable(),
-                          util::getRSSBytes()};
+                          util::getRSSBytes(),
+                          slabsApproxFreePercentages};
 }
 
 template <typename CacheTrait>

cachelib/allocator/CacheAllocator.h

@@ -1320,6 +1320,8 @@ class CacheAllocator : public CacheBase {
 #pragma GCC diagnostic pop
 
  private:
+  double slabsApproxFreePercentage(TierId tid) const;
+
   // wrapper around Item's refcount and active handle tracking
   FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(Item& it, bool failIfMoving);
   FOLLY_ALWAYS_INLINE RefcountWithFlags::Value decRef(Item& it);

cachelib/allocator/CacheStats.h

@@ -554,6 +554,9 @@ struct CacheMemoryStats {
 
   // returne usable portion of the cache size
   size_t usableRamCacheSize() const { return ramCacheSize; }
+
+  // percentage of free slabs
+  std::vector<double> slabsApproxFreePercentages{0.0};
 };
 
 // Stats for compact cache

cachelib/allocator/memory/AllocationClass.cpp

@@ -51,6 +51,7 @@ AllocationClass::AllocationClass(ClassId classId,
       allocationSize_(allocSize),
       slabAlloc_(s),
       freedAllocations_{slabAlloc_.createSingleTierPtrCompressor<FreeAlloc>()} {
+  curAllocatedSlabs_ = allocatedSlabs_.size();
   checkState();
 }
 
@@ -87,6 +88,12 @@ void AllocationClass::checkState() const {
         "Current allocation slab {} is not in allocated slabs list",
         currSlab_));
   }
+
+  if (curAllocatedSlabs_ != allocatedSlabs_.size()) {
+    throw std::invalid_argument(folly::sformat(
+      "Mismatch in allocated slabs numbers"
+    ));
+  }
 }
 
 // TODO(stuclar): Add poolId to the metadata to be serialized when cache shuts
@@ -116,10 +123,12 @@ AllocationClass::AllocationClass(
     freeSlabs_.push_back(slabAlloc_.getSlabForIdx(freeSlabIdx));
   }
 
+  curAllocatedSlabs_ = allocatedSlabs_.size();
   checkState();
 }
 
 void AllocationClass::addSlabLocked(Slab* slab) {
+  curAllocatedSlabs_.fetch_add(1, std::memory_order_relaxed);
   canAllocate_ = true;
   auto header = slabAlloc_.getSlabHeader(slab);
   header->classId = classId_;
@@ -168,6 +177,7 @@ void* AllocationClass::allocateLocked() {
   }
 
   XDCHECK(canAllocate_);
+  curAllocatedSize_.fetch_add(getAllocSize(), std::memory_order_relaxed);
 
   // grab from the free list if possible.
   if (!freedAllocations_.empty()) {
@@ -270,6 +280,7 @@ SlabReleaseContext AllocationClass::startSlabRelease(
                          slab, getId()));
     }
     *allocIt = allocatedSlabs_.back();
+    curAllocatedSlabs_.fetch_sub(1, std::memory_order_relaxed);
     allocatedSlabs_.pop_back();
 
     // if slab is being carved currently, then update slabReleaseAllocMap
@@ -511,6 +522,7 @@ void AllocationClass::abortSlabRelease(const SlabReleaseContext& context) {
     }
     slabReleaseAllocMap_.erase(slabPtrVal);
     allocatedSlabs_.push_back(const_cast<Slab*>(slab));
+    curAllocatedSlabs_.fetch_add(1, std::memory_order_relaxed);
     // restore the classId and allocSize
     header->classId = classId_;
     header->allocSize = allocationSize_;
@@ -661,6 +673,8 @@ void AllocationClass::free(void* memory) {
     freedAllocations_.insert(*reinterpret_cast<FreeAlloc*>(memory));
     canAllocate_ = true;
   });
+
+  curAllocatedSize_.fetch_sub(getAllocSize(), std::memory_order_relaxed);
 }
 
 serialization::AllocationClassObject AllocationClass::saveState() const {
@@ -699,8 +713,8 @@ ACStats AllocationClass::getStats() const {
     const unsigned long long nFreedAllocs = freedAllocations_.size();
     const unsigned long long nActiveAllocs =
         nSlabsAllocated * perSlab - nFreedAllocs - freeAllocsInCurrSlab;
-    return {allocationSize_, perSlab,       nSlabsAllocated, freeSlabs_.size(),
-            nFreedAllocs,    nActiveAllocs, isFull()};
+    return {allocationSize_, perSlab, 0, nSlabsAllocated, freeSlabs_.size(),
+            nFreedAllocs, nActiveAllocs, isFull(), util::RollingStats{}, 0.0};
   });
 }
 
@@ -723,3 +737,12 @@ std::vector<bool>& AllocationClass::getSlabReleaseAllocMapLocked(
   const auto slabPtrVal = getSlabPtrValue(slab);
   return slabReleaseAllocMap_.at(slabPtrVal);
 }
+
+double AllocationClass::approxFreePercentage() const {
+  if (getNumSlabs() == 0) {
+    return 100.0;
+  }
+
+  return 100.0 - 100.0 * static_cast<double>(curAllocatedSize_.load(std::memory_order_relaxed)) /
+    static_cast<double>(getNumSlabs() * Slab::kSize);
+}

cachelib/allocator/memory/AllocationClass.h

@@ -94,6 +94,11 @@ class AllocationClass {
     return static_cast<unsigned int>(Slab::kSize / allocationSize_);
   }
 
+  // total number of slabs under this AllocationClass.
+  unsigned int getNumSlabs() const {
+    return curAllocatedSlabs_.load(std::memory_order_relaxed);
+  }
+
   // fetch stats about this allocation class.
   ACStats getStats() const;
 
@@ -308,6 +313,9 @@ class AllocationClass {
   // @throw std::logic_error if the object state can not be serialized
   serialization::AllocationClassObject saveState() const;
 
+  // approximate percent of free memory inside this allocation class
+  double approxFreePercentage() const;
+
  private:
   // check if the state of the AllocationClass is valid and if not, throws an
   // std::invalid_argument exception. This is intended for use in
@@ -467,6 +475,12 @@ class AllocationClass {
 
   std::atomic<int64_t> activeReleases_{0};
 
+  // amount of memory currently allocated by this AC
+  std::atomic<size_t> curAllocatedSize_{0};
+
+  // total number of slabs under this AllocationClass.
+  std::atomic<size_t> curAllocatedSlabs_{0};
+
   // stores the list of outstanding allocations for a given slab. This is
   // created when we start a slab release process and if there are any active
   // allocaitons need to be marked as free.

cachelib/allocator/memory/MemoryAllocator.h

@@ -416,6 +416,14 @@ class MemoryAllocator {
     return memoryPoolManager_.getPoolIds();
   }
 
+  double approxFreeSlabsPercentage() const {
+    if (slabAllocator_.getNumUsableAndAdvisedSlabs() == 0)
+      return 100.0;
+  
+    return 100.0 - 100.0 * static_cast<double>(slabAllocator_.approxNumSlabsAllocated()) /
+     slabAllocator_.getNumUsableAndAdvisedSlabs();
+  }
+
   // fetches the memory pool for the id if one exists. This is purely to get
   // information out of the pool.
   //

cachelib/allocator/memory/MemoryAllocatorStats.h

@@ -33,6 +33,9 @@ struct ACStats {
   // number of allocations per slab
   unsigned long long allocsPerSlab;
 
+  // size of memory assigned to this allocation class
+  unsigned long long memorySize;
+
   // number of slabs that are currently used for active allocations.
   unsigned long long usedSlabs;
 
@@ -51,6 +54,9 @@ struct ACStats {
   // Rolling allocation latency (in ns)
   util::RollingStats allocLatencyNs;
 
+  // percent of free memory in this class
+  double approxFreePercent{0.0};
+
   constexpr unsigned long long totalSlabs() const noexcept {
     return freeSlabs + usedSlabs;
   }

cachelib/allocator/memory/SlabAllocator.cpp

@@ -360,6 +360,8 @@ Slab* SlabAllocator::makeNewSlab(PoolId id) {
     return nullptr;
   }
 
+  numSlabsAllocated_.fetch_add(1, std::memory_order_relaxed);
+
   memoryPoolSize_[id] += sizeof(Slab);
   // initialize the header for the slab.
   initializeHeader(slab, id);
@@ -375,6 +377,8 @@ void SlabAllocator::freeSlab(Slab* slab) {
   }
 
   memoryPoolSize_[header->poolId] -= sizeof(Slab);
+  numSlabsAllocated_.fetch_sub(1, std::memory_order_relaxed);
+
   // grab the lock
   LockHolder l(lock_);
   freeSlabs_.push_back(slab);

cachelib/allocator/memory/SlabAllocator.h

@@ -329,7 +329,13 @@ class SlabAllocator {
                                    memorySize_);
   }
 
- private:
+  size_t approxNumSlabsAllocated() const {
+    return numSlabsAllocated_.load(std::memory_order_relaxed);
+  }
+
+private:
+  std::atomic<size_t> numSlabsAllocated_{0};
+
   // null Slab* presenttation. With 4M Slab size, a valid slab index would never
   // reach 2^16 - 1;
   static constexpr SlabIdx kNullSlabIdx = std::numeric_limits<SlabIdx>::max();

cachelib/cachebench/cache/Cache-inl.h

@@ -634,6 +634,7 @@ Stats Cache<Allocator>::getStats() const {
   const auto rebalanceStats = cache_->getSlabReleaseStats();
   const auto navyStats = cache_->getNvmCacheStatsMap().toMap();
 
+  ret.slabsApproxFreePercentages = cache_->getCacheMemoryStats().slabsApproxFreePercentages;
   ret.allocationClassStats = allocationClassStats;
   ret.numEvictions = aggregate.numEvictions();
   ret.numItems = aggregate.numItems();