resizable_memory: release pages with MADV_DONTNEED on linux

Author: pknowles

include/decodeless/detail/mappedfile_linux.hpp

@@ -88,6 +88,7 @@ class MemoryMap {
     static constexpr bool ProtNone = MemoryProtection == PROT_NONE;
     static constexpr bool Writable = (MemoryProtection & PROT_WRITE) != 0;
     using address_type = std::conditional_t<Writable || ProtNone, void*, const void*>;
+    using byte_type = std::conditional_t<Writable || ProtNone, std::byte, const std::byte>;
     MemoryMap(address_type addr, size_t length, int flags, int fd, off_t offset)
         : m_size(length)
         , m_address(mmap(const_cast<void*>(addr), length, MemoryProtection, flags, fd, offset))
@@ -123,6 +124,9 @@ class MemoryMap {
         unmap();
     }
     address_type address() const { return m_address; }
+    address_type address(size_t offset) const {
+        return static_cast<address_type>(static_cast<byte_type*>(m_address) + offset);
+    }
     size_t       size() const { return m_size; }
     void         sync(int flags = MS_SYNC | MS_INVALIDATE)
         requires Writable
@@ -135,12 +139,7 @@ class MemoryMap {
             throw LastError();
     }
     void resize(size_t size) {
-#if 0
-        void* addr = mremap(m_address, m_size, size,
-                            MREMAP_MAYMOVE | MREMAP_FIXED | MREMAP_DONTUNMAP, m_address);
-#else
         void* addr = mremap(m_address, m_size, size, 0);
-#endif
         if (addr == MAP_FAILED) {
             throw LastError();
         } else if (addr != m_address) {
@@ -268,23 +267,34 @@ class ResizableMappedMemory {
         size_t ps = pageSize();
         size_t newMappedSize = ((size + ps - 1) / ps) * ps;
 
-        // Add/remove just the new range
-        if (newMappedSize > m_mappedSize)
-            protect(m_mappedSize, newMappedSize - m_mappedSize, PROT_READ | PROT_WRITE);
-        else if (newMappedSize < m_mappedSize)
-            protect(newMappedSize, m_mappedSize - newMappedSize, PROT_NONE);
+        if (newMappedSize > m_mappedSize) {
+            // Add just the new range
+            if (mprotect(m_reserved.address(m_mappedSize), newMappedSize - m_mappedSize,
+                         PROT_READ | PROT_WRITE) != 0)
+                throw LastError();
+        } else if (newMappedSize < m_mappedSize) {
+            // Release unused range to the OS. mprotect() will not do this.
+            // Using MADV_DONTNEED is many times faster than mmap().
+#if 1
+            if (mprotect(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize,
+                         PROT_NONE) != 0)
+                throw LastError();
+            if (madvise(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize,
+                        MADV_DONTNEED) != 0)
+                throw LastError();
+#else
+            if (mmap(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize, PROT_NONE,
+                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1, 0) == MAP_FAILED)
+                throw LastError();
+#endif
+        }
         m_mappedSize = newMappedSize;
         m_size = size;
     }
 
     ResizableMappedMemory& operator=(ResizableMappedMemory&& other) noexcept = default;
 
 private:
-    void protect(size_t offset, size_t size, int prot) {
-        if (mprotect(reinterpret_cast<void*>(uintptr_t(m_reserved.address()) + offset), size,
-                     prot) != 0)
-            throw LastError();
-    }
     detail::MemoryMap<PROT_NONE> m_reserved;
     size_t                       m_size = 0;
     size_t                       m_mappedSize = 0;

test/src/mappedfile.cpp

@@ -1,10 +1,11 @@
 // Copyright (c) 2024 Pyarelal Knowles, MIT License
 
+#include <algorithm>
 #include <cstdio>
+#include <decodeless/mappedfile.hpp>
 #include <filesystem>
 #include <fstream>
 #include <gtest/gtest.h>
-#include <decodeless/mappedfile.hpp>
 #include <ostream>
 #include <span>
 
@@ -226,7 +227,7 @@ TEST_F(MappedFileFixture, LinuxResize) {
 
 #endif
 
-TEST_F(MappedFileFixture, ResizeMemory) {
+TEST(MappedMemory, ResizeMemory) {
     const char str[] = "hello world!";
     {
         resizable_memory file(0, 10000);
@@ -270,7 +271,7 @@ TEST_F(MappedFileFixture, ResizeMemory) {
     }
 }
 
-TEST_F(MappedFileFixture, ResizeMemoryExtended) {
+TEST(MappedMemory, ResizeMemoryExtended) {
     size_t           nextBytes = 1;
     resizable_memory memory(nextBytes, 1llu << 32); // 4gb of virtual memory pls
     void*            data = memory.data();
@@ -425,3 +426,58 @@ TEST_F(MappedFileFixture, Readme) {
     fs::remove(tmpFile2);
     EXPECT_FALSE(fs::exists(tmpFile2));
 }
+
+#ifndef _WIN32
+std::vector<uint8_t> getResidency(void* base, size_t size) {
+    std::vector<unsigned char> result(size / getpagesize(), 0u);
+    int                        ret = mincore(base, size, result.data());
+    if (ret != 0)
+        throw detail::LastError();
+    return result;
+}
+
+TEST(MappedMemory, PageResidencyAfterDecommit) {
+    const size_t page_size = getpagesize();
+    const size_t reserve_size = page_size * 64; // 64 pages total
+    const size_t commit_size = page_size * 4;   // We'll use 4 pages
+
+    // Reserve virtual address space (uncommitted, inaccessible)
+    void* base =
+        mmap(nullptr, reserve_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+    ASSERT_NE(base, MAP_FAILED) << "Failed to mmap reserved space";
+    EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size),
+                                    [](uint8_t c) { return (c & 1u) == 0; }));
+
+    // Commit a portion with PROT_READ | PROT_WRITE
+    int prot_result = mprotect(base, commit_size, PROT_READ | PROT_WRITE);
+    ASSERT_EQ(prot_result, 0) << "Failed to mprotect committed region";
+    EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size),
+                                    [](uint8_t c) { return (c & 1u) == 0; }));
+
+    // Touch the memory to ensure it's backed by RAM
+    std::span committed(static_cast<std::byte*>(base), commit_size);
+    std::ranges::fill(committed, std::byte(0xAB));
+
+    // Verify pages are resident using mincore
+    EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size),
+                                    [](uint8_t c) { return (c & 1u) == 1; }));
+
+    // Decommit
+    #if 0
+    void* remap = mmap(base, commit_size, PROT_NONE,
+                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1, 0);
+    ASSERT_EQ(remap, base) << "Failed to remap to decommit pages";
+    #else
+    // See MADV_FREE discussion here: https://github.com/golang/go/issues/42330
+    prot_result = mprotect(base, commit_size, PROT_NONE);
+    ASSERT_EQ(prot_result, 0) << "Failed to mprotect committed region back to PROT_NONE";
+    int madvise_result = madvise(base, commit_size, MADV_DONTNEED);
+    ASSERT_EQ(madvise_result, 0) << "Failed to release pages with madvise";
+    #endif
+    EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size),
+                                    [](uint8_t c) { return (c & 1u) == 0; }));
+
+    // Cleanup
+    munmap(base, reserve_size);
+}
+#endif